lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "log"
  16         "math/rand"
  17         "net/http"
  18         "net/http/httptest"
  19         "os"
  20         "os/exec"
  21         "regexp"
  22         "runtime/pprof"
  23         "strings"
  24         "sync"
  25         "sync/atomic"
  26         "syscall"
  27         "testing"
  28         "time"
  29
  30         "git.arvados.org/arvados.git/lib/cloud"
  31         "git.arvados.org/arvados.git/lib/cmd"
  32         "git.arvados.org/arvados.git/sdk/go/arvados"
  33         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  34         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  35         "git.arvados.org/arvados.git/sdk/go/manifest"
  36         "golang.org/x/net/context"
  37
  38         . "gopkg.in/check.v1"
  39 )
  40
  41 // Gocheck boilerplate
  42 func TestCrunchExec(t *testing.T) {
  43         TestingT(t)
  44 }
  45
  46 var _ = Suite(&TestSuite{})
  47
  48 type TestSuite struct {
  49         client                   *arvados.Client
  50         api                      *ArvTestClient
  51         runner                   *ContainerRunner
  52         executor                 *stubExecutor
  53         keepmount                string
  54         keepmountTmp             []string
  55         testDispatcherKeepClient KeepTestClient
  56         testContainerKeepClient  KeepTestClient
  57 }
  58
  59 func (s *TestSuite) SetUpTest(c *C) {
  60         s.client = arvados.NewClientFromEnv()
  61         s.executor = &stubExecutor{}
  62         var err error
  63         s.api = &ArvTestClient{}
  64         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  65         c.Assert(err, IsNil)
  66         s.runner.executor = s.executor
  67         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  68                 return s.api, &s.testContainerKeepClient, s.client, nil
  69         }
  70         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  71                 s.runner.ArvMountPoint = s.keepmount
  72                 for i, opt := range cmd {
  73                         if opt == "--mount-tmp" {
  74                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  75                                 if err != nil {
  76                                         return nil, err
  77                                 }
  78                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  79                         }
  80                 }
  81                 return nil, nil
  82         }
  83         s.keepmount = c.MkDir()
  84         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  85         s.keepmountTmp = nil
  86         c.Assert(err, IsNil)
  87         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  88         c.Assert(err, IsNil)
  89         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  90         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  91         c.Assert(err, IsNil)
  92         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  93         c.Assert(err, IsNil)
  94         s.runner.ArvMountPoint = s.keepmount
  95 }
  96
  97 type ArvTestClient struct {
  98         Total   int64
  99         Calls   int
 100         Content []arvadosclient.Dict
 101         arvados.Container
 102         secretMounts []byte
 103         Logs         map[string]*bytes.Buffer
 104         sync.Mutex
 105         WasSetRunning bool
 106         callraw       bool
 107 }
 108
 109 type KeepTestClient struct {
 110         Called         bool
 111         Content        []byte
 112         StorageClasses []string
 113 }
 114
 115 type stubExecutor struct {
 116         imageLoaded bool
 117         loaded      string
 118         loadErr     error
 119         exitCode    int
 120         createErr   error
 121         created     containerSpec
 122         startErr    error
 123         waitSleep   time.Duration
 124         waitErr     error
 125         stopErr     error
 126         stopped     bool
 127         closed      bool
 128         runFunc     func() int
 129         exit        chan int
 130 }
 131
 132 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 133         containerClient *arvados.Client) error {
 134         e.loaded = tarball
 135         return e.loadErr
 136 }
 137 func (e *stubExecutor) Runtime() string                 { return "stub" }
 138 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 139 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 140 func (e *stubExecutor) Start() error {
 141         e.exit = make(chan int, 1)
 142         go func() { e.exit <- e.runFunc() }()
 143         return e.startErr
 144 }
 145 func (e *stubExecutor) CgroupID() string { return "cgroupid" }
 146 func (e *stubExecutor) Stop() error      { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 147 func (e *stubExecutor) Close()           { e.closed = true }
 148 func (e *stubExecutor) Wait(context.Context) (int, error) {
 149         return <-e.exit, e.waitErr
 150 }
 151 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 152         return nil, errors.New("unimplemented")
 153 }
 154 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 155
 156 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 157
 158 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 159 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 160 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 161
 162 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 163 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 164
 165 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 166 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 167 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 168 `
 169
 170 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 171
 172 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 173 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 174
 175 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 176 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 177
 178 func (client *ArvTestClient) Create(resourceType string,
 179         parameters arvadosclient.Dict,
 180         output interface{}) error {
 181
 182         client.Mutex.Lock()
 183         defer client.Mutex.Unlock()
 184
 185         client.Calls++
 186         client.Content = append(client.Content, parameters)
 187
 188         if resourceType == "logs" {
 189                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 190                 if client.Logs == nil {
 191                         client.Logs = make(map[string]*bytes.Buffer)
 192                 }
 193                 if client.Logs[et] == nil {
 194                         client.Logs[et] = &bytes.Buffer{}
 195                 }
 196                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 197         }
 198
 199         if resourceType == "collections" && output != nil {
 200                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 201                 md5sum := md5.Sum([]byte(mt))
 202                 outmap := output.(*arvados.Collection)
 203                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 204                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 205         }
 206
 207         return nil
 208 }
 209
 210 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 211         switch {
 212         case method == "GET" && resourceType == "containers" && action == "auth":
 213                 return json.Unmarshal([]byte(`{
 214                         "kind": "arvados#api_client_authorization",
 215                         "uuid": "`+fakeAuthUUID+`",
 216                         "api_token": "`+fakeAuthToken+`"
 217                         }`), output)
 218         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 219                 if client.secretMounts != nil {
 220                         return json.Unmarshal(client.secretMounts, output)
 221                 }
 222                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 223         default:
 224                 return fmt.Errorf("Not found")
 225         }
 226 }
 227
 228 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 229         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 230         var j []byte
 231         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 232                 j = []byte(`{
 233                         "kind": "arvados#nodeList",
 234                         "items": [{
 235                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 236                                 "hostname": "compute2",
 237                                 "properties": {"total_cpu_cores": 16}
 238                         }]}`)
 239         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 240                 if uuid == "" {
 241                         j, err = json.Marshal(map[string]interface{}{
 242                                 "items": []interface{}{client.Container},
 243                                 "kind":  "arvados#nodeList",
 244                         })
 245                 } else {
 246                         j, err = json.Marshal(client.Container)
 247                 }
 248         } else {
 249                 j = []byte(`{
 250                         "command": ["sleep", "1"],
 251                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 252                         "cwd": ".",
 253                         "environment": {},
 254                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 255                         "output_path": "/tmp",
 256                         "priority": 1,
 257                         "runtime_constraints": {}
 258                 }`)
 259         }
 260         return ioutil.NopCloser(bytes.NewReader(j)), err
 261 }
 262
 263 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 264         if resourceType == "collections" {
 265                 if uuid == hwPDH {
 266                         output.(*arvados.Collection).ManifestText = hwManifest
 267                 } else if uuid == otherPDH {
 268                         output.(*arvados.Collection).ManifestText = otherManifest
 269                 } else if uuid == normalizedWithSubdirsPDH {
 270                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 271                 } else if uuid == denormalizedWithSubdirsPDH {
 272                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 273                 }
 274         }
 275         if resourceType == "containers" {
 276                 (*output.(*arvados.Container)) = client.Container
 277         }
 278         return nil
 279 }
 280
 281 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 282         client.Mutex.Lock()
 283         defer client.Mutex.Unlock()
 284         client.Calls++
 285         client.Content = append(client.Content, parameters)
 286         if resourceType == "containers" {
 287                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 288                         client.WasSetRunning = true
 289                 }
 290         } else if resourceType == "collections" && output != nil {
 291                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 292                 output.(*arvados.Collection).UUID = uuid
 293                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 294         }
 295         return nil
 296 }
 297
 298 var discoveryMap = map[string]interface{}{
 299         "defaultTrashLifetime":               float64(1209600),
 300         "crunchLimitLogBytesPerJob":          float64(67108864),
 301         "crunchLogThrottleBytes":             float64(65536),
 302         "crunchLogThrottlePeriod":            float64(60),
 303         "crunchLogThrottleLines":             float64(1024),
 304         "crunchLogPartialLineThrottlePeriod": float64(5),
 305         "crunchLogBytesPerEvent":             float64(4096),
 306         "crunchLogSecondsBetweenEvents":      float64(1),
 307 }
 308
 309 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 310         return discoveryMap[key], nil
 311 }
 312
 313 // CalledWith returns the parameters from the first API call whose
 314 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 315 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 316 // no call matches, it returns nil.
 317 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 318 call:
 319         for _, content := range client.Content {
 320                 var v interface{} = content
 321                 for _, k := range strings.Split(jpath, ".") {
 322                         if dict, ok := v.(arvadosclient.Dict); !ok {
 323                                 continue call
 324                         } else {
 325                                 v = dict[k]
 326                         }
 327                 }
 328                 if v == expect {
 329                         return content
 330                 }
 331         }
 332         return nil
 333 }
 334
 335 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 336         return locator, nil
 337 }
 338
 339 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 340         client.Content = opts.Data
 341         return arvados.BlockWriteResponse{
 342                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 343         }, nil
 344 }
 345
 346 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 347         return 0, errors.New("not implemented")
 348 }
 349
 350 func (client *KeepTestClient) ClearBlockCache() {
 351 }
 352
 353 func (client *KeepTestClient) Close() {
 354         client.Content = nil
 355 }
 356
 357 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 358         client.StorageClasses = sc
 359 }
 360
 361 type FileWrapper struct {
 362         io.ReadCloser
 363         len int64
 364 }
 365
 366 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 367         return nil, errors.New("not implemented")
 368 }
 369
 370 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 371         return 0, errors.New("not implemented")
 372 }
 373
 374 func (fw FileWrapper) Size() int64 {
 375         return fw.len
 376 }
 377
 378 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 379         return nil, errors.New("not implemented")
 380 }
 381
 382 func (fw FileWrapper) Truncate(int64) error {
 383         return errors.New("not implemented")
 384 }
 385
 386 func (fw FileWrapper) Write([]byte) (int, error) {
 387         return 0, errors.New("not implemented")
 388 }
 389
 390 func (fw FileWrapper) Sync() error {
 391         return errors.New("not implemented")
 392 }
 393
 394 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 395         return nil, errors.New("not implemented")
 396 }
 397
 398 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 399         return errors.New("not implemented")
 400 }
 401
 402 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 403         if filename == hwImageID+".tar" {
 404                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 405                 client.Called = true
 406                 return FileWrapper{rdr, 1321984}, nil
 407         } else if filename == "/file1_in_main.txt" {
 408                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 409                 client.Called = true
 410                 return FileWrapper{rdr, 3}, nil
 411         }
 412         return nil, nil
 413 }
 414
 415 func (s *TestSuite) TestLoadImage(c *C) {
 416         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 417         s.runner.Container.Mounts = map[string]arvados.Mount{
 418                 "/out": {Kind: "tmp", Writable: true},
 419         }
 420         s.runner.Container.OutputPath = "/out"
 421
 422         _, err := s.runner.SetupMounts()
 423         c.Assert(err, IsNil)
 424
 425         imageID, err := s.runner.LoadImage()
 426         c.Check(err, IsNil)
 427         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 428         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 429
 430         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 431         s.executor.imageLoaded = false
 432         s.executor.loaded = ""
 433         s.executor.loadErr = errors.New("bork")
 434         imageID, err = s.runner.LoadImage()
 435         c.Check(err, ErrorMatches, ".*bork")
 436         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 437
 438         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 439         s.executor.imageLoaded = false
 440         s.executor.loaded = ""
 441         s.executor.loadErr = nil
 442         imageID, err = s.runner.LoadImage()
 443         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 444         c.Check(s.executor.loaded, Equals, "")
 445 }
 446
 447 type ArvErrorTestClient struct{}
 448
 449 func (ArvErrorTestClient) Create(resourceType string,
 450         parameters arvadosclient.Dict,
 451         output interface{}) error {
 452         return nil
 453 }
 454
 455 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 456         if method == "GET" && resourceType == "containers" && action == "auth" {
 457                 return nil
 458         }
 459         return errors.New("ArvError")
 460 }
 461
 462 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 463         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 464         return nil, errors.New("ArvError")
 465 }
 466
 467 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 468         return errors.New("ArvError")
 469 }
 470
 471 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 472         return nil
 473 }
 474
 475 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 476         return discoveryMap[key], nil
 477 }
 478
 479 type KeepErrorTestClient struct {
 480         KeepTestClient
 481 }
 482
 483 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 484         return nil, errors.New("KeepError")
 485 }
 486
 487 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 488         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 489 }
 490
 491 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 492         return "", errors.New("KeepError")
 493 }
 494
 495 type KeepReadErrorTestClient struct {
 496         KeepTestClient
 497 }
 498
 499 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 500         return 0, errors.New("KeepError")
 501 }
 502
 503 type ErrorReader struct {
 504         FileWrapper
 505 }
 506
 507 func (ErrorReader) Read(p []byte) (n int, err error) {
 508         return 0, errors.New("ErrorReader")
 509 }
 510
 511 func (ErrorReader) Seek(int64, int) (int64, error) {
 512         return 0, errors.New("ErrorReader")
 513 }
 514
 515 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 516         return ErrorReader{}, nil
 517 }
 518
 519 type ClosableBuffer struct {
 520         bytes.Buffer
 521 }
 522
 523 func (*ClosableBuffer) Close() error {
 524         return nil
 525 }
 526
 527 type TestLogs struct {
 528         Stdout ClosableBuffer
 529         Stderr ClosableBuffer
 530 }
 531
 532 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 533         if logstr == "stdout" {
 534                 return &tl.Stdout, nil
 535         }
 536         if logstr == "stderr" {
 537                 return &tl.Stderr, nil
 538         }
 539         return nil, errors.New("???")
 540 }
 541
 542 func dockerLog(fd byte, msg string) []byte {
 543         by := []byte(msg)
 544         header := make([]byte, 8+len(by))
 545         header[0] = fd
 546         header[7] = byte(len(by))
 547         copy(header[8:], by)
 548         return header
 549 }
 550
 551 func (s *TestSuite) TestRunContainer(c *C) {
 552         s.executor.runFunc = func() int {
 553                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 554                 return 0
 555         }
 556
 557         var logs TestLogs
 558         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 559         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 560         s.runner.Container.Command = []string{"./hw"}
 561         s.runner.Container.OutputStorageClasses = []string{"default"}
 562
 563         imageID, err := s.runner.LoadImage()
 564         c.Assert(err, IsNil)
 565
 566         err = s.runner.CreateContainer(imageID, nil)
 567         c.Assert(err, IsNil)
 568
 569         err = s.runner.StartContainer()
 570         c.Assert(err, IsNil)
 571
 572         err = s.runner.WaitFinish()
 573         c.Assert(err, IsNil)
 574
 575         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 576         c.Check(logs.Stderr.String(), Equals, "")
 577 }
 578
 579 func (s *TestSuite) TestCommitLogs(c *C) {
 580         api := &ArvTestClient{}
 581         kc := &KeepTestClient{}
 582         defer kc.Close()
 583         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 584         c.Assert(err, IsNil)
 585         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 586
 587         cr.CrunchLog.Print("Hello world!")
 588         cr.CrunchLog.Print("Goodbye")
 589         cr.finalState = "Complete"
 590
 591         err = cr.CommitLogs()
 592         c.Check(err, IsNil)
 593
 594         c.Check(api.Calls, Equals, 2)
 595         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 596         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 597         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 598         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 599 }
 600
 601 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 602         api := &ArvTestClient{}
 603         kc := &KeepTestClient{}
 604         defer kc.Close()
 605         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 606         c.Assert(err, IsNil)
 607
 608         err = cr.UpdateContainerRunning("")
 609         c.Check(err, IsNil)
 610
 611         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 612 }
 613
 614 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 615         api := &ArvTestClient{}
 616         kc := &KeepTestClient{}
 617         defer kc.Close()
 618         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 619         c.Assert(err, IsNil)
 620
 621         cr.LogsPDH = new(string)
 622         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 623
 624         cr.ExitCode = new(int)
 625         *cr.ExitCode = 42
 626         cr.finalState = "Complete"
 627
 628         err = cr.UpdateContainerFinal()
 629         c.Check(err, IsNil)
 630
 631         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 632         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 633         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 634 }
 635
 636 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 637         api := &ArvTestClient{}
 638         kc := &KeepTestClient{}
 639         defer kc.Close()
 640         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 641         c.Assert(err, IsNil)
 642         cr.cCancelled = true
 643         cr.finalState = "Cancelled"
 644
 645         err = cr.UpdateContainerFinal()
 646         c.Check(err, IsNil)
 647
 648         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 649         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 650         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 651 }
 652
 653 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 654 // dress rehearsal of the Run() function, starting from a JSON container record.
 655 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, fn func() int) (*ArvTestClient, *ContainerRunner, string) {
 656         err := json.Unmarshal([]byte(record), &s.api.Container)
 657         c.Assert(err, IsNil)
 658         initialState := s.api.Container.State
 659
 660         var sm struct {
 661                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 662         }
 663         err = json.Unmarshal([]byte(record), &sm)
 664         c.Check(err, IsNil)
 665         secretMounts, err := json.Marshal(sm)
 666         c.Assert(err, IsNil)
 667         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 668
 669         s.executor.runFunc = fn
 670
 671         s.runner.statInterval = 100 * time.Millisecond
 672         s.runner.containerWatchdogInterval = time.Second
 673
 674         realTemp := c.MkDir()
 675         tempcount := 0
 676         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 677                 tempcount++
 678                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 679                 err := os.Mkdir(d, os.ModePerm)
 680                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 681                         // Test case must have pre-populated the tempdir
 682                         err = nil
 683                 }
 684                 return d, err
 685         }
 686         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 687                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 688         }
 689
 690         if extraMounts != nil && len(extraMounts) > 0 {
 691                 err := s.runner.SetupArvMountPoint("keep")
 692                 c.Check(err, IsNil)
 693
 694                 for _, m := range extraMounts {
 695                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 696                 }
 697         }
 698
 699         err = s.runner.Run()
 700         if s.api.CalledWith("container.state", "Complete") != nil {
 701                 c.Check(err, IsNil)
 702         }
 703         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 704                 c.Check(s.api.WasSetRunning, Equals, true)
 705                 var lastupdate arvadosclient.Dict
 706                 for _, content := range s.api.Content {
 707                         if content["container"] != nil {
 708                                 lastupdate = content["container"].(arvadosclient.Dict)
 709                         }
 710                 }
 711                 if lastupdate["log"] == nil {
 712                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 713                 }
 714         }
 715
 716         if err != nil {
 717                 for k, v := range s.api.Logs {
 718                         c.Log(k)
 719                         c.Log(v.String())
 720                 }
 721         }
 722
 723         return s.api, s.runner, realTemp
 724 }
 725
 726 func (s *TestSuite) TestFullRunHello(c *C) {
 727         s.runner.enableMemoryLimit = true
 728         s.runner.networkMode = "default"
 729         s.fullRunHelper(c, `{
 730     "command": ["echo", "hello world"],
 731     "container_image": "`+arvadostest.DockerImage112PDH+`",
 732     "cwd": ".",
 733     "environment": {"foo":"bar","baz":"waz"},
 734     "mounts": {"/tmp": {"kind": "tmp"} },
 735     "output_path": "/tmp",
 736     "priority": 1,
 737     "runtime_constraints": {"vcpus":1,"ram":1000000},
 738     "state": "Locked",
 739     "output_storage_classes": ["default"]
 740 }`, nil, func() int {
 741                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 742                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 743                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 744                 c.Check(s.executor.created.VCPUs, Equals, 1)
 745                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 746                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 747                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 748                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 749                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 750                 return 0
 751         })
 752
 753         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 754         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 755         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 756         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 757         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 758 }
 759
 760 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 761         var ran bool
 762         s.fullRunHelper(c, `{
 763     "command": ["sleep", "3"],
 764     "container_image": "`+arvadostest.DockerImage112PDH+`",
 765     "cwd": ".",
 766     "environment": {},
 767     "mounts": {"/tmp": {"kind": "tmp"} },
 768     "output_path": "/tmp",
 769     "priority": 1,
 770     "runtime_constraints": {},
 771     "scheduling_parameters":{"max_run_time": 1},
 772     "state": "Running"
 773 }`, nil, func() int {
 774                 ran = true
 775                 return 2
 776         })
 777         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 778         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 779         c.Check(ran, Equals, false)
 780 }
 781
 782 func ec2MetadataServerStub(c *C, token *string, failureRate float64, stoptime *atomic.Value) *httptest.Server {
 783         failedOnce := false
 784         return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 785                 if !failedOnce || rand.Float64() < failureRate {
 786                         w.WriteHeader(http.StatusServiceUnavailable)
 787                         failedOnce = true
 788                         return
 789                 }
 790                 switch r.URL.Path {
 791                 case "/latest/api/token":
 792                         fmt.Fprintln(w, *token)
 793                 case "/latest/meta-data/spot/instance-action":
 794                         if r.Header.Get("X-aws-ec2-metadata-token") != *token {
 795                                 w.WriteHeader(http.StatusUnauthorized)
 796                         } else if t, _ := stoptime.Load().(time.Time); t.IsZero() {
 797                                 w.WriteHeader(http.StatusNotFound)
 798                         } else {
 799                                 fmt.Fprintf(w, `{"action":"stop","time":"%s"}`, t.Format(time.RFC3339))
 800                         }
 801                 default:
 802                         w.WriteHeader(http.StatusNotFound)
 803                 }
 804         }))
 805 }
 806
 807 func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
 808         s.testSpotInterruptionNotice(c, 0.1)
 809 }
 810
 811 func (s *TestSuite) TestSpotInterruptionNoticeNotAvailable(c *C) {
 812         s.testSpotInterruptionNotice(c, 1)
 813 }
 814
 815 func (s *TestSuite) testSpotInterruptionNotice(c *C, failureRate float64) {
 816         var stoptime atomic.Value
 817         token := "fake-ec2-metadata-token"
 818         stub := ec2MetadataServerStub(c, &token, failureRate, &stoptime)
 819         defer stub.Close()
 820
 821         defer func(i time.Duration, u string) {
 822                 spotInterruptionCheckInterval = i
 823                 ec2MetadataBaseURL = u
 824         }(spotInterruptionCheckInterval, ec2MetadataBaseURL)
 825         spotInterruptionCheckInterval = time.Second / 8
 826         ec2MetadataBaseURL = stub.URL
 827
 828         go s.runner.checkSpotInterruptionNotices()
 829         s.fullRunHelper(c, `{
 830     "command": ["sleep", "3"],
 831     "container_image": "`+arvadostest.DockerImage112PDH+`",
 832     "cwd": ".",
 833     "environment": {},
 834     "mounts": {"/tmp": {"kind": "tmp"} },
 835     "output_path": "/tmp",
 836     "priority": 1,
 837     "runtime_constraints": {},
 838     "state": "Locked"
 839 }`, nil, func() int {
 840                 time.Sleep(time.Second)
 841                 stoptime.Store(time.Now().Add(time.Minute).UTC())
 842                 token = "different-fake-ec2-metadata-token"
 843                 time.Sleep(time.Second)
 844                 return 0
 845         })
 846         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Checking for spot interruptions every 125ms using instance metadata at http://.*`)
 847         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Error checking spot interruptions: 503 Service Unavailable.*`)
 848         if failureRate == 1 {
 849                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Giving up on checking spot interruptions after too many consecutive failures.*`)
 850         } else {
 851                 text := `Cloud provider indicates instance action "stop" scheduled for time "` + stoptime.Load().(time.Time).Format(time.RFC3339) + `"`
 852                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*`+text+`.*`)
 853                 c.Check(s.api.CalledWith("container.runtime_status.warning", "preemption notice"), NotNil)
 854                 c.Check(s.api.CalledWith("container.runtime_status.warningDetail", text), NotNil)
 855                 c.Check(s.api.CalledWith("container.runtime_status.preemptionNotice", text), NotNil)
 856         }
 857 }
 858
 859 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 860         s.fullRunHelper(c, `{
 861     "command": ["sleep", "3"],
 862     "container_image": "`+arvadostest.DockerImage112PDH+`",
 863     "cwd": ".",
 864     "environment": {},
 865     "mounts": {"/tmp": {"kind": "tmp"} },
 866     "output_path": "/tmp",
 867     "priority": 1,
 868     "runtime_constraints": {},
 869     "scheduling_parameters":{"max_run_time": 1},
 870     "state": "Locked"
 871 }`, nil, func() int {
 872                 time.Sleep(3 * time.Second)
 873                 return 0
 874         })
 875
 876         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 877         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 878 }
 879
 880 func (s *TestSuite) TestContainerWaitFails(c *C) {
 881         s.fullRunHelper(c, `{
 882     "command": ["sleep", "3"],
 883     "container_image": "`+arvadostest.DockerImage112PDH+`",
 884     "cwd": ".",
 885     "mounts": {"/tmp": {"kind": "tmp"} },
 886     "output_path": "/tmp",
 887     "priority": 1,
 888     "state": "Locked"
 889 }`, nil, func() int {
 890                 s.executor.waitErr = errors.New("Container is not running")
 891                 return 0
 892         })
 893
 894         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 895         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 896 }
 897
 898 func (s *TestSuite) TestCrunchstat(c *C) {
 899         s.fullRunHelper(c, `{
 900                 "command": ["sleep", "1"],
 901                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 902                 "cwd": ".",
 903                 "environment": {},
 904                 "mounts": {"/tmp": {"kind": "tmp"} },
 905                 "output_path": "/tmp",
 906                 "priority": 1,
 907                 "runtime_constraints": {},
 908                 "state": "Locked"
 909         }`, nil, func() int {
 910                 time.Sleep(time.Second)
 911                 return 0
 912         })
 913
 914         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 915         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 916
 917         // We didn't actually start a container, so crunchstat didn't
 918         // find accounting files and therefore didn't log any stats.
 919         // It should have logged a "can't find accounting files"
 920         // message after one poll interval, though, so we can confirm
 921         // it's alive:
 922         c.Assert(s.api.Logs["crunchstat"], NotNil)
 923         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 924
 925         // The "files never appeared" log assures us that we called
 926         // (*crunchstat.Reporter)Stop(), and that we set it up with
 927         // the correct container ID "abcde":
 928         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 929 }
 930
 931 func (s *TestSuite) TestNodeInfoLog(c *C) {
 932         os.Setenv("SLURMD_NODENAME", "compute2")
 933         s.fullRunHelper(c, `{
 934                 "command": ["sleep", "1"],
 935                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 936                 "cwd": ".",
 937                 "environment": {},
 938                 "mounts": {"/tmp": {"kind": "tmp"} },
 939                 "output_path": "/tmp",
 940                 "priority": 1,
 941                 "runtime_constraints": {},
 942                 "state": "Locked"
 943         }`, nil, func() int {
 944                 time.Sleep(time.Second)
 945                 return 0
 946         })
 947
 948         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 949         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 950
 951         c.Assert(s.api.Logs["node"], NotNil)
 952         json := s.api.Logs["node"].String()
 953         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 954         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 955         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 956
 957         c.Assert(s.api.Logs["node-info"], NotNil)
 958         json = s.api.Logs["node-info"].String()
 959         c.Check(json, Matches, `(?ms).*Host Information.*`)
 960         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 961         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 962         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 963         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 964 }
 965
 966 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 967         s.fullRunHelper(c, `{
 968                 "command": ["sleep", "1"],
 969                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 970                 "cwd": ".",
 971                 "environment": {},
 972                 "mounts": {"/tmp": {"kind": "tmp"} },
 973                 "output_path": "/tmp",
 974                 "priority": 1,
 975                 "runtime_constraints": {},
 976                 "state": "Locked"
 977         }`, nil, func() int {
 978                 return 0
 979         })
 980
 981         c.Assert(s.api.Logs["crunch-run"], NotNil)
 982         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 983         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 984         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 985         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 986 }
 987
 988 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
 989         var collection_create, container_update arvadosclient.Dict
 990         s.fullRunHelper(c, `{
 991                 "command": ["true"],
 992                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 993                 "cwd": ".",
 994                 "environment": {},
 995                 "mounts": {"/tmp": {"kind": "tmp"} },
 996                 "output_path": "/tmp",
 997                 "priority": 1,
 998                 "runtime_constraints": {},
 999                 "state": "Locked",
1000                 "uuid": "zzzzz-dz642-202301121543210"
1001         }`, nil, func() int {
1002                 collection_create = s.api.CalledWith("ensure_unique_name", true)
1003                 container_update = s.api.CalledWith("container.state", "Running")
1004                 return 0
1005         })
1006
1007         c.Assert(collection_create, NotNil)
1008         log_collection := collection_create["collection"].(arvadosclient.Dict)
1009         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
1010         manifest_text := log_collection["manifest_text"].(string)
1011         // We check that the file size is at least two digits as an easy way to
1012         // check the file isn't empty.
1013         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node-info\.txt( .+)?\n`)
1014         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node\.json( .+)?\n`)
1015
1016         c.Assert(container_update, NotNil)
1017         // As of Arvados 2.5.0, the container update must specify its log in PDH
1018         // format for the API server to propagate it to container requests, which
1019         // is what we care about for this test.
1020         expect_pdh := fmt.Sprintf("%x+%d", md5.Sum([]byte(manifest_text)), len(manifest_text))
1021         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Equals, expect_pdh)
1022 }
1023
1024 func (s *TestSuite) TestContainerRecordLog(c *C) {
1025         s.fullRunHelper(c, `{
1026                 "command": ["sleep", "1"],
1027                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1028                 "cwd": ".",
1029                 "environment": {},
1030                 "mounts": {"/tmp": {"kind": "tmp"} },
1031                 "output_path": "/tmp",
1032                 "priority": 1,
1033                 "runtime_constraints": {},
1034                 "state": "Locked"
1035         }`, nil,
1036                 func() int {
1037                         time.Sleep(time.Second)
1038                         return 0
1039                 })
1040
1041         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1042         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1043
1044         c.Assert(s.api.Logs["container"], NotNil)
1045         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
1046 }
1047
1048 func (s *TestSuite) TestFullRunStderr(c *C) {
1049         s.fullRunHelper(c, `{
1050     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
1051     "container_image": "`+arvadostest.DockerImage112PDH+`",
1052     "cwd": ".",
1053     "environment": {},
1054     "mounts": {"/tmp": {"kind": "tmp"} },
1055     "output_path": "/tmp",
1056     "priority": 1,
1057     "runtime_constraints": {},
1058     "state": "Locked"
1059 }`, nil, func() int {
1060                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1061                 fmt.Fprintln(s.executor.created.Stderr, "world")
1062                 return 1
1063         })
1064
1065         final := s.api.CalledWith("container.state", "Complete")
1066         c.Assert(final, NotNil)
1067         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1068         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1069
1070         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
1071         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
1072 }
1073
1074 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
1075         s.fullRunHelper(c, `{
1076     "command": ["pwd"],
1077     "container_image": "`+arvadostest.DockerImage112PDH+`",
1078     "cwd": ".",
1079     "environment": {},
1080     "mounts": {"/tmp": {"kind": "tmp"} },
1081     "output_path": "/tmp",
1082     "priority": 1,
1083     "runtime_constraints": {},
1084     "state": "Locked"
1085 }`, nil, func() int {
1086                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
1087                 return 0
1088         })
1089
1090         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1091         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1092         c.Log(s.api.Logs["stdout"])
1093         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
1094 }
1095
1096 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1097         s.fullRunHelper(c, `{
1098     "command": ["pwd"],
1099     "container_image": "`+arvadostest.DockerImage112PDH+`",
1100     "cwd": "/bin",
1101     "environment": {},
1102     "mounts": {"/tmp": {"kind": "tmp"} },
1103     "output_path": "/tmp",
1104     "priority": 1,
1105     "runtime_constraints": {},
1106     "state": "Locked"
1107 }`, nil, func() int {
1108                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1109                 return 0
1110         })
1111
1112         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1113         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1114         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1115 }
1116
1117 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1118         s.fullRunHelper(c, `{
1119     "command": ["pwd"],
1120     "container_image": "`+arvadostest.DockerImage112PDH+`",
1121     "cwd": "/bin",
1122     "environment": {},
1123     "mounts": {"/tmp": {"kind": "tmp"} },
1124     "output_path": "/tmp",
1125     "priority": 1,
1126     "runtime_constraints": {},
1127     "state": "Locked",
1128     "output_storage_classes": ["foo", "bar"]
1129 }`, nil, func() int {
1130                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1131                 return 0
1132         })
1133
1134         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1135         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1136         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1137         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1138         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1139 }
1140
1141 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1142         s.fullRunHelper(c, `{
1143     "command": ["pwd"],
1144     "container_image": "`+arvadostest.DockerImage112PDH+`",
1145     "cwd": "/bin",
1146     "environment": {},
1147     "mounts": {"/tmp": {"kind": "tmp"} },
1148     "output_path": "/tmp",
1149     "priority": 1,
1150     "runtime_constraints": {"cuda": {"device_count": 2}},
1151     "state": "Locked",
1152     "output_storage_classes": ["foo", "bar"]
1153 }`, nil, func() int {
1154                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1155                 return 0
1156         })
1157         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1158 }
1159
1160 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1161         s.fullRunHelper(c, `{
1162     "command": ["pwd"],
1163     "container_image": "`+arvadostest.DockerImage112PDH+`",
1164     "cwd": "/bin",
1165     "environment": {},
1166     "mounts": {"/tmp": {"kind": "tmp"} },
1167     "output_path": "/tmp",
1168     "priority": 1,
1169     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1170     "state": "Locked",
1171     "output_storage_classes": ["foo", "bar"]
1172 }`, nil, func() int {
1173                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1174                 return 0
1175         })
1176         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1177 }
1178
1179 func (s *TestSuite) TestStopOnSignal(c *C) {
1180         s.executor.runFunc = func() int {
1181                 s.executor.created.Stdout.Write([]byte("foo\n"))
1182                 s.runner.SigChan <- syscall.SIGINT
1183                 time.Sleep(10 * time.Second)
1184                 return 0
1185         }
1186         s.testStopContainer(c)
1187 }
1188
1189 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1190         s.executor.runFunc = func() int {
1191                 s.executor.created.Stdout.Write([]byte("foo\n"))
1192                 s.runner.ArvMountExit <- nil
1193                 close(s.runner.ArvMountExit)
1194                 time.Sleep(10 * time.Second)
1195                 return 0
1196         }
1197         s.runner.ArvMountExit = make(chan error)
1198         s.testStopContainer(c)
1199 }
1200
1201 func (s *TestSuite) testStopContainer(c *C) {
1202         record := `{
1203     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1204     "container_image": "` + arvadostest.DockerImage112PDH + `",
1205     "cwd": ".",
1206     "environment": {},
1207     "mounts": {"/tmp": {"kind": "tmp"} },
1208     "output_path": "/tmp",
1209     "priority": 1,
1210     "runtime_constraints": {},
1211     "state": "Locked"
1212 }`
1213
1214         err := json.Unmarshal([]byte(record), &s.api.Container)
1215         c.Assert(err, IsNil)
1216
1217         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1218         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1219                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1220         }
1221
1222         done := make(chan error)
1223         go func() {
1224                 done <- s.runner.Run()
1225         }()
1226         select {
1227         case <-time.After(20 * time.Second):
1228                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1229                 c.Fatal("timed out")
1230         case err = <-done:
1231                 c.Check(err, IsNil)
1232         }
1233         for k, v := range s.api.Logs {
1234                 c.Log(k)
1235                 c.Log(v.String(), "\n")
1236         }
1237
1238         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1239         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1240         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1241 }
1242
1243 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1244         s.fullRunHelper(c, `{
1245     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1246     "container_image": "`+arvadostest.DockerImage112PDH+`",
1247     "cwd": "/bin",
1248     "environment": {"FROBIZ": "bilbo"},
1249     "mounts": {"/tmp": {"kind": "tmp"} },
1250     "output_path": "/tmp",
1251     "priority": 1,
1252     "runtime_constraints": {},
1253     "state": "Locked"
1254 }`, nil, func() int {
1255                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1256                 return 0
1257         })
1258
1259         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1260         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1261         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1262 }
1263
1264 type ArvMountCmdLine struct {
1265         Cmd   []string
1266         token string
1267 }
1268
1269 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1270         am.Cmd = c
1271         am.token = token
1272         return nil, nil
1273 }
1274
1275 func stubCert(temp string) string {
1276         path := temp + "/ca-certificates.crt"
1277         crt, _ := os.Create(path)
1278         crt.Close()
1279         arvadosclient.CertFiles = []string{path}
1280         return path
1281 }
1282
1283 func (s *TestSuite) TestSetupMounts(c *C) {
1284         cr := s.runner
1285         am := &ArvMountCmdLine{}
1286         cr.RunArvMount = am.ArvMountTest
1287         cr.ContainerArvClient = &ArvTestClient{}
1288         cr.ContainerKeepClient = &KeepTestClient{}
1289         cr.Container.OutputStorageClasses = []string{"default"}
1290
1291         realTemp := c.MkDir()
1292         certTemp := c.MkDir()
1293         stubCertPath := stubCert(certTemp)
1294         cr.parentTemp = realTemp
1295
1296         i := 0
1297         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1298                 i++
1299                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1300                 err := os.Mkdir(d, os.ModePerm)
1301                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1302                         // Test case must have pre-populated the tempdir
1303                         err = nil
1304                 }
1305                 return d, err
1306         }
1307
1308         checkEmpty := func() {
1309                 // Should be deleted.
1310                 _, err := os.Stat(realTemp)
1311                 c.Assert(os.IsNotExist(err), Equals, true)
1312
1313                 // Now recreate it for the next test.
1314                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1315         }
1316
1317         {
1318                 i = 0
1319                 cr.ArvMountPoint = ""
1320                 cr.Container.Mounts = make(map[string]arvados.Mount)
1321                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1322                 cr.Container.OutputPath = "/tmp"
1323                 cr.statInterval = 5 * time.Second
1324                 bindmounts, err := cr.SetupMounts()
1325                 c.Check(err, IsNil)
1326                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1327                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1328                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1329                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1330                 os.RemoveAll(cr.ArvMountPoint)
1331                 cr.CleanupDirs()
1332                 checkEmpty()
1333         }
1334
1335         {
1336                 i = 0
1337                 cr.ArvMountPoint = ""
1338                 cr.Container.Mounts = make(map[string]arvados.Mount)
1339                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1340                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1341                 cr.Container.OutputPath = "/out"
1342                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1343
1344                 bindmounts, err := cr.SetupMounts()
1345                 c.Check(err, IsNil)
1346                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1347                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1348                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1349                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1350                 os.RemoveAll(cr.ArvMountPoint)
1351                 cr.CleanupDirs()
1352                 checkEmpty()
1353         }
1354
1355         {
1356                 i = 0
1357                 cr.ArvMountPoint = ""
1358                 cr.Container.Mounts = make(map[string]arvados.Mount)
1359                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1360                 cr.Container.OutputPath = "/tmp"
1361                 cr.Container.RuntimeConstraints.API = true
1362                 cr.Container.OutputStorageClasses = []string{"default"}
1363
1364                 bindmounts, err := cr.SetupMounts()
1365                 c.Check(err, IsNil)
1366                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1367                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1368                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1369                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1370                 os.RemoveAll(cr.ArvMountPoint)
1371                 cr.CleanupDirs()
1372                 checkEmpty()
1373
1374                 cr.Container.RuntimeConstraints.API = false
1375         }
1376
1377         {
1378                 i = 0
1379                 cr.ArvMountPoint = ""
1380                 cr.Container.Mounts = map[string]arvados.Mount{
1381                         "/keeptmp": {Kind: "collection", Writable: true},
1382                 }
1383                 cr.Container.OutputPath = "/keeptmp"
1384
1385                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1386
1387                 bindmounts, err := cr.SetupMounts()
1388                 c.Check(err, IsNil)
1389                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1390                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1391                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1392                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1393                 os.RemoveAll(cr.ArvMountPoint)
1394                 cr.CleanupDirs()
1395                 checkEmpty()
1396         }
1397
1398         {
1399                 i = 0
1400                 cr.ArvMountPoint = ""
1401                 cr.Container.Mounts = map[string]arvados.Mount{
1402                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1403                         "/keepout": {Kind: "collection", Writable: true},
1404                 }
1405                 cr.Container.OutputPath = "/keepout"
1406
1407                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1408                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1409
1410                 bindmounts, err := cr.SetupMounts()
1411                 c.Check(err, IsNil)
1412                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1413                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1414                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1415                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1416                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1417                         "/keepout": {realTemp + "/keep1/tmp0", false},
1418                 })
1419                 os.RemoveAll(cr.ArvMountPoint)
1420                 cr.CleanupDirs()
1421                 checkEmpty()
1422         }
1423
1424         {
1425                 i = 0
1426                 cr.ArvMountPoint = ""
1427                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1428                 cr.Container.Mounts = map[string]arvados.Mount{
1429                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1430                         "/keepout": {Kind: "collection", Writable: true},
1431                 }
1432                 cr.Container.OutputPath = "/keepout"
1433
1434                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1435                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1436
1437                 bindmounts, err := cr.SetupMounts()
1438                 c.Check(err, IsNil)
1439                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1440                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1441                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1442                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1443                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1444                         "/keepout": {realTemp + "/keep1/tmp0", false},
1445                 })
1446                 os.RemoveAll(cr.ArvMountPoint)
1447                 cr.CleanupDirs()
1448                 checkEmpty()
1449         }
1450
1451         for _, test := range []struct {
1452                 in  interface{}
1453                 out string
1454         }{
1455                 {in: "foo", out: `"foo"`},
1456                 {in: nil, out: `null`},
1457                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1458         } {
1459                 i = 0
1460                 cr.ArvMountPoint = ""
1461                 cr.Container.Mounts = map[string]arvados.Mount{
1462                         "/mnt/test.json": {Kind: "json", Content: test.in},
1463                 }
1464                 bindmounts, err := cr.SetupMounts()
1465                 c.Check(err, IsNil)
1466                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1467                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1468                 })
1469                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1470                 c.Check(err, IsNil)
1471                 c.Check(content, DeepEquals, []byte(test.out))
1472                 os.RemoveAll(cr.ArvMountPoint)
1473                 cr.CleanupDirs()
1474                 checkEmpty()
1475         }
1476
1477         for _, test := range []struct {
1478                 in  interface{}
1479                 out string
1480         }{
1481                 {in: "foo", out: `foo`},
1482                 {in: nil, out: "error"},
1483                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1484         } {
1485                 i = 0
1486                 cr.ArvMountPoint = ""
1487                 cr.Container.Mounts = map[string]arvados.Mount{
1488                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1489                 }
1490                 bindmounts, err := cr.SetupMounts()
1491                 if test.out == "error" {
1492                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1493                 } else {
1494                         c.Check(err, IsNil)
1495                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1496                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1497                         })
1498                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1499                         c.Check(err, IsNil)
1500                         c.Check(content, DeepEquals, []byte(test.out))
1501                 }
1502                 os.RemoveAll(cr.ArvMountPoint)
1503                 cr.CleanupDirs()
1504                 checkEmpty()
1505         }
1506
1507         // Read-only mount points are allowed underneath output_dir mount point
1508         {
1509                 i = 0
1510                 cr.ArvMountPoint = ""
1511                 cr.Container.Mounts = make(map[string]arvados.Mount)
1512                 cr.Container.Mounts = map[string]arvados.Mount{
1513                         "/tmp":     {Kind: "tmp"},
1514                         "/tmp/foo": {Kind: "collection"},
1515                 }
1516                 cr.Container.OutputPath = "/tmp"
1517
1518                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1519
1520                 bindmounts, err := cr.SetupMounts()
1521                 c.Check(err, IsNil)
1522                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1523                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1524                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1525                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1526                         "/tmp":     {realTemp + "/tmp2", false},
1527                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1528                 })
1529                 os.RemoveAll(cr.ArvMountPoint)
1530                 cr.CleanupDirs()
1531                 checkEmpty()
1532         }
1533
1534         // Writable mount points copied to output_dir mount point
1535         {
1536                 i = 0
1537                 cr.ArvMountPoint = ""
1538                 cr.Container.Mounts = make(map[string]arvados.Mount)
1539                 cr.Container.Mounts = map[string]arvados.Mount{
1540                         "/tmp": {Kind: "tmp"},
1541                         "/tmp/foo": {Kind: "collection",
1542                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1543                                 Writable:         true},
1544                         "/tmp/bar": {Kind: "collection",
1545                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1546                                 Path:             "baz",
1547                                 Writable:         true},
1548                 }
1549                 cr.Container.OutputPath = "/tmp"
1550
1551                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1552                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1553
1554                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1555                 rf.Write([]byte("bar"))
1556                 rf.Close()
1557
1558                 _, err := cr.SetupMounts()
1559                 c.Check(err, IsNil)
1560                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1561                 c.Check(err, IsNil)
1562                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1563                 c.Check(err, IsNil)
1564                 os.RemoveAll(cr.ArvMountPoint)
1565                 cr.CleanupDirs()
1566                 checkEmpty()
1567         }
1568
1569         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1570         {
1571                 i = 0
1572                 cr.ArvMountPoint = ""
1573                 cr.Container.Mounts = make(map[string]arvados.Mount)
1574                 cr.Container.Mounts = map[string]arvados.Mount{
1575                         "/tmp":     {Kind: "tmp"},
1576                         "/tmp/foo": {Kind: "tmp"},
1577                 }
1578                 cr.Container.OutputPath = "/tmp"
1579
1580                 _, err := cr.SetupMounts()
1581                 c.Check(err, NotNil)
1582                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1583                 os.RemoveAll(cr.ArvMountPoint)
1584                 cr.CleanupDirs()
1585                 checkEmpty()
1586         }
1587
1588         // Only mount point of kind 'collection' is allowed for stdin
1589         {
1590                 i = 0
1591                 cr.ArvMountPoint = ""
1592                 cr.Container.Mounts = make(map[string]arvados.Mount)
1593                 cr.Container.Mounts = map[string]arvados.Mount{
1594                         "stdin": {Kind: "tmp"},
1595                 }
1596
1597                 _, err := cr.SetupMounts()
1598                 c.Check(err, NotNil)
1599                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1600                 os.RemoveAll(cr.ArvMountPoint)
1601                 cr.CleanupDirs()
1602                 checkEmpty()
1603         }
1604
1605         // git_tree mounts
1606         {
1607                 i = 0
1608                 cr.ArvMountPoint = ""
1609                 (*GitMountSuite)(nil).useTestGitServer(c)
1610                 cr.token = arvadostest.ActiveToken
1611                 cr.Container.Mounts = make(map[string]arvados.Mount)
1612                 cr.Container.Mounts = map[string]arvados.Mount{
1613                         "/tip": {
1614                                 Kind:   "git_tree",
1615                                 UUID:   arvadostest.Repository2UUID,
1616                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1617                                 Path:   "/",
1618                         },
1619                         "/non-tip": {
1620                                 Kind:   "git_tree",
1621                                 UUID:   arvadostest.Repository2UUID,
1622                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1623                                 Path:   "/",
1624                         },
1625                 }
1626                 cr.Container.OutputPath = "/tmp"
1627
1628                 bindmounts, err := cr.SetupMounts()
1629                 c.Check(err, IsNil)
1630
1631                 for path, mount := range bindmounts {
1632                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1633                 }
1634
1635                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1636                 c.Check(err, IsNil)
1637                 c.Check(string(data), Equals, "\000\001\002\003")
1638                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1639                 c.Check(err, FitsTypeOf, &os.PathError{})
1640                 c.Check(os.IsNotExist(err), Equals, true)
1641
1642                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1643                 c.Check(err, IsNil)
1644                 c.Check(string(data), Equals, "\000\001\002\003")
1645                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1646                 c.Check(err, IsNil)
1647                 c.Check(string(data), Equals, "testfile\n")
1648
1649                 cr.CleanupDirs()
1650                 checkEmpty()
1651         }
1652 }
1653
1654 func (s *TestSuite) TestStdout(c *C) {
1655         helperRecord := `{
1656                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1657                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1658                 "cwd": "/bin",
1659                 "environment": {"FROBIZ": "bilbo"},
1660                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1661                 "output_path": "/tmp",
1662                 "priority": 1,
1663                 "runtime_constraints": {},
1664                 "state": "Locked"
1665         }`
1666
1667         s.fullRunHelper(c, helperRecord, nil, func() int {
1668                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1669                 return 0
1670         })
1671
1672         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1673         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1674         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1675 }
1676
1677 // Used by the TestStdoutWithWrongPath*()
1678 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func() int) (*ArvTestClient, *ContainerRunner, error) {
1679         err := json.Unmarshal([]byte(record), &s.api.Container)
1680         c.Assert(err, IsNil)
1681         s.executor.runFunc = fn
1682         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1683         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1684                 return s.api, &KeepTestClient{}, nil, nil
1685         }
1686         return s.api, s.runner, s.runner.Run()
1687 }
1688
1689 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1690         _, _, err := s.stdoutErrorRunHelper(c, `{
1691     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1692     "output_path": "/tmp",
1693     "state": "Locked"
1694 }`, func() int { return 0 })
1695         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1696 }
1697
1698 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1699         _, _, err := s.stdoutErrorRunHelper(c, `{
1700     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1701     "output_path": "/tmp",
1702     "state": "Locked"
1703 }`, func() int { return 0 })
1704         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1705 }
1706
1707 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1708         _, _, err := s.stdoutErrorRunHelper(c, `{
1709     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1710     "output_path": "/tmp",
1711     "state": "Locked"
1712 }`, func() int { return 0 })
1713         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1714 }
1715
1716 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1717         s.fullRunHelper(c, `{
1718     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1719     "container_image": "`+arvadostest.DockerImage112PDH+`",
1720     "cwd": "/bin",
1721     "environment": {},
1722     "mounts": {"/tmp": {"kind": "tmp"} },
1723     "output_path": "/tmp",
1724     "priority": 1,
1725     "runtime_constraints": {"API": true},
1726     "state": "Locked"
1727 }`, nil, func() int {
1728                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1729                 return 3
1730         })
1731         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1732         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1733         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1734 }
1735
1736 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1737         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1738         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1739         s.fullRunHelper(c, `{
1740     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1741     "container_image": "`+arvadostest.DockerImage112PDH+`",
1742     "cwd": "/bin",
1743     "environment": {},
1744     "mounts": {"/tmp": {"kind": "tmp"} },
1745     "output_path": "/tmp",
1746     "priority": 1,
1747     "runtime_constraints": {"API": true},
1748     "state": "Locked"
1749 }`, nil, func() int {
1750                 s.api.Container.Output = arvadostest.DockerImage112PDH
1751                 return 0
1752         })
1753
1754         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1755         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1756         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1757 }
1758
1759 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1760         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1761                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1762                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1763                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1764         }
1765         s.executor.runFunc = func() int {
1766                 time.Sleep(time.Second)
1767                 return 137
1768         }
1769         record := `{
1770     "command": ["sleep", "1"],
1771     "container_image": "` + arvadostest.DockerImage112PDH + `",
1772     "cwd": "/bin",
1773     "environment": {},
1774     "mounts": {"/tmp": {"kind": "tmp"} },
1775     "output_path": "/tmp",
1776     "priority": 1,
1777     "runtime_constraints": {"API": true},
1778     "state": "Locked"
1779 }`
1780         err := json.Unmarshal([]byte(record), &s.api.Container)
1781         c.Assert(err, IsNil)
1782         err = s.runner.Run()
1783         c.Assert(err, IsNil)
1784         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1785         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1786         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1787         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1788         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1789 }
1790
1791 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1792         helperRecord := `{
1793                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1794                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1795                 "cwd": "/bin",
1796                 "environment": {"FROBIZ": "bilbo"},
1797                 "mounts": {
1798         "/tmp": {"kind": "tmp"},
1799         "/tmp/foo": {"kind": "collection",
1800                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1801                      "exclude_from_output": true
1802         },
1803         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1804     },
1805                 "output_path": "/tmp",
1806                 "priority": 1,
1807                 "runtime_constraints": {},
1808                 "state": "Locked"
1809         }`
1810
1811         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1812
1813         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1814                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1815                 return 0
1816         })
1817
1818         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1819         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1820         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1821 }
1822
1823 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1824         helperRecord := `{
1825                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1826                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1827                 "cwd": "/bin",
1828                 "environment": {"FROBIZ": "bilbo"},
1829                 "mounts": {
1830         "/tmp": {"kind": "tmp"},
1831         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1832         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1833         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1834         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1835         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1836     },
1837                 "output_path": "/tmp",
1838                 "priority": 1,
1839                 "runtime_constraints": {},
1840                 "state": "Locked",
1841                 "uuid": "zzzzz-dz642-202301130848001"
1842         }`
1843
1844         extraMounts := []string{
1845                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1846                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1847                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1848         }
1849
1850         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1851                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1852                 return 0
1853         })
1854
1855         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1856                 "/tmp":                   {realtemp + "/tmp1", false},
1857                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1858                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1859                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1860                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1861         })
1862
1863         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1864         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1865         output_count := uint(0)
1866         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1867                 if v["collection"] == nil {
1868                         continue
1869                 }
1870                 collection := v["collection"].(arvadosclient.Dict)
1871                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
1872                         continue
1873                 }
1874                 c.Check(v["ensure_unique_name"], Equals, true)
1875                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1876 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1877 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1878 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1879 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1880 `)
1881                 output_count++
1882         }
1883         c.Check(output_count, Not(Equals), uint(0))
1884 }
1885
1886 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1887         helperRecord := `{
1888                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1889                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1890                 "cwd": "/bin",
1891                 "environment": {"FROBIZ": "bilbo"},
1892                 "mounts": {
1893         "/tmp": {"kind": "tmp"},
1894         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1895         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1896     },
1897                 "output_path": "/tmp",
1898                 "priority": 1,
1899                 "runtime_constraints": {},
1900                 "state": "Locked",
1901                 "uuid": "zzzzz-dz642-202301130848002"
1902         }`
1903
1904         extraMounts := []string{
1905                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1906         }
1907
1908         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1909                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1910                 return 0
1911         })
1912
1913         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1914         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1915         output_count := uint(0)
1916         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1917                 if v["collection"] == nil {
1918                         continue
1919                 }
1920                 collection := v["collection"].(arvadosclient.Dict)
1921                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
1922                         continue
1923                 }
1924                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1925 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1926 `)
1927                 output_count++
1928         }
1929         c.Check(output_count, Not(Equals), uint(0))
1930 }
1931
1932 func (s *TestSuite) TestOutputError(c *C) {
1933         helperRecord := `{
1934                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1935                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1936                 "cwd": "/bin",
1937                 "environment": {"FROBIZ": "bilbo"},
1938                 "mounts": {
1939                         "/tmp": {"kind": "tmp"}
1940                 },
1941                 "output_path": "/tmp",
1942                 "priority": 1,
1943                 "runtime_constraints": {},
1944                 "state": "Locked"
1945         }`
1946         s.fullRunHelper(c, helperRecord, nil, func() int {
1947                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1948                 return 0
1949         })
1950
1951         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1952 }
1953
1954 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1955         helperRecord := `{
1956                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1957                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1958                 "cwd": "/bin",
1959                 "environment": {"FROBIZ": "bilbo"},
1960                 "mounts": {
1961         "/tmp": {"kind": "tmp"},
1962         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1963         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1964     },
1965                 "output_path": "/tmp",
1966                 "priority": 1,
1967                 "runtime_constraints": {},
1968                 "state": "Locked"
1969         }`
1970
1971         extraMounts := []string{
1972                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1973         }
1974
1975         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1976                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1977                 return 0
1978         })
1979
1980         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1981         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1982         for _, v := range api.Content {
1983                 if v["collection"] != nil {
1984                         collection := v["collection"].(arvadosclient.Dict)
1985                         if strings.Index(collection["name"].(string), "output") == 0 {
1986                                 manifest := collection["manifest_text"].(string)
1987                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1988 `)
1989                         }
1990                 }
1991         }
1992 }
1993
1994 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1995         helperRecord := `{
1996                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1997                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1998                 "cwd": "/bin",
1999                 "environment": {"FROBIZ": "bilbo"},
2000                 "mounts": {
2001         "/tmp": {"kind": "tmp"},
2002         "stdin": {"kind": "json", "content": "foo"},
2003         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2004     },
2005                 "output_path": "/tmp",
2006                 "priority": 1,
2007                 "runtime_constraints": {},
2008                 "state": "Locked"
2009         }`
2010
2011         api, _, _ := s.fullRunHelper(c, helperRecord, nil, func() int {
2012                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2013                 return 0
2014         })
2015
2016         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2017         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2018         for _, v := range api.Content {
2019                 if v["collection"] != nil {
2020                         collection := v["collection"].(arvadosclient.Dict)
2021                         if strings.Index(collection["name"].(string), "output") == 0 {
2022                                 manifest := collection["manifest_text"].(string)
2023                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2024 `)
2025                         }
2026                 }
2027         }
2028 }
2029
2030 func (s *TestSuite) TestStderrMount(c *C) {
2031         api, cr, _ := s.fullRunHelper(c, `{
2032     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
2033     "container_image": "`+arvadostest.DockerImage112PDH+`",
2034     "cwd": ".",
2035     "environment": {},
2036     "mounts": {"/tmp": {"kind": "tmp"},
2037                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
2038                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
2039     "output_path": "/tmp",
2040     "priority": 1,
2041     "runtime_constraints": {},
2042     "state": "Locked"
2043 }`, nil, func() int {
2044                 fmt.Fprintln(s.executor.created.Stdout, "hello")
2045                 fmt.Fprintln(s.executor.created.Stderr, "oops")
2046                 return 1
2047         })
2048
2049         final := api.CalledWith("container.state", "Complete")
2050         c.Assert(final, NotNil)
2051         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
2052         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
2053
2054         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
2055 }
2056
2057 func (s *TestSuite) TestNumberRoundTrip(c *C) {
2058         s.api.callraw = true
2059         err := s.runner.fetchContainerRecord()
2060         c.Assert(err, IsNil)
2061         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
2062         c.Logf("%#v", s.runner.Container)
2063         c.Check(err, IsNil)
2064         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
2065 }
2066
2067 func (s *TestSuite) TestFullBrokenDocker(c *C) {
2068         nextState := ""
2069         for _, setup := range []func(){
2070                 func() {
2071                         c.Log("// waitErr = ocl runtime error")
2072                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
2073                         nextState = "Cancelled"
2074                 },
2075                 func() {
2076                         c.Log("// loadErr = cannot connect")
2077                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
2078                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
2079                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
2080                         c.Assert(err, IsNil)
2081                         nextState = "Queued"
2082                 },
2083         } {
2084                 s.SetUpTest(c)
2085                 setup()
2086                 s.fullRunHelper(c, `{
2087     "command": ["echo", "hello world"],
2088     "container_image": "`+arvadostest.DockerImage112PDH+`",
2089     "cwd": ".",
2090     "environment": {},
2091     "mounts": {"/tmp": {"kind": "tmp"} },
2092     "output_path": "/tmp",
2093     "priority": 1,
2094     "runtime_constraints": {},
2095     "state": "Locked"
2096 }`, nil, func() int { return 0 })
2097                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
2098                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
2099                 if s.runner.brokenNodeHook != "" {
2100                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
2101                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
2102                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2103                 } else {
2104                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2105                 }
2106         }
2107 }
2108
2109 func (s *TestSuite) TestBadCommand(c *C) {
2110         for _, startError := range []string{
2111                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
2112                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
2113                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2114         } {
2115                 s.SetUpTest(c)
2116                 s.executor.startErr = errors.New(startError)
2117                 s.fullRunHelper(c, `{
2118     "command": ["echo", "hello world"],
2119     "container_image": "`+arvadostest.DockerImage112PDH+`",
2120     "cwd": ".",
2121     "environment": {},
2122     "mounts": {"/tmp": {"kind": "tmp"} },
2123     "output_path": "/tmp",
2124     "priority": 1,
2125     "runtime_constraints": {},
2126     "state": "Locked"
2127 }`, nil, func() int { return 0 })
2128                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2129                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2130         }
2131 }
2132
2133 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2134         helperRecord := `{
2135                 "command": ["true"],
2136                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2137                 "cwd": "/bin",
2138                 "mounts": {
2139                     "/tmp": {"kind": "tmp"},
2140                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2141                 },
2142                 "secret_mounts": {
2143                 },
2144                 "output_path": "/tmp",
2145                 "priority": 1,
2146                 "runtime_constraints": {},
2147                 "state": "Locked"
2148         }`
2149
2150         s.fullRunHelper(c, helperRecord, nil, func() int {
2151                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2152                 c.Check(err, IsNil)
2153                 c.Check(string(content), Equals, "mypassword")
2154                 return 0
2155         })
2156
2157         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2158         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2159         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2160         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2161
2162         // under secret mounts, not captured in output
2163         helperRecord = `{
2164                 "command": ["true"],
2165                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2166                 "cwd": "/bin",
2167                 "mounts": {
2168                     "/tmp": {"kind": "tmp"}
2169                 },
2170                 "secret_mounts": {
2171                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2172                 },
2173                 "output_path": "/tmp",
2174                 "priority": 1,
2175                 "runtime_constraints": {},
2176                 "state": "Locked"
2177         }`
2178
2179         s.SetUpTest(c)
2180         s.fullRunHelper(c, helperRecord, nil, func() int {
2181                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2182                 c.Check(err, IsNil)
2183                 c.Check(string(content), Equals, "mypassword")
2184                 return 0
2185         })
2186
2187         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2188         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2189         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2190         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2191
2192         // under secret mounts, output dir is a collection, not captured in output
2193         helperRecord = `{
2194                 "command": ["true"],
2195                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2196                 "cwd": "/bin",
2197                 "mounts": {
2198                     "/tmp": {"kind": "collection", "writable": true}
2199                 },
2200                 "secret_mounts": {
2201                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2202                 },
2203                 "output_path": "/tmp",
2204                 "priority": 1,
2205                 "runtime_constraints": {},
2206                 "state": "Locked"
2207         }`
2208
2209         s.SetUpTest(c)
2210         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, func() int {
2211                 // secret.conf should be provisioned as a separate
2212                 // bind mount, i.e., it should not appear in the
2213                 // (fake) fuse filesystem as viewed from the host.
2214                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2215                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2216                         c.Logf("secret.conf: content %q, err %#v", content, err)
2217                 }
2218                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2219                 c.Check(err, IsNil)
2220                 return 0
2221         })
2222
2223         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2224         c.Check(err, IsNil)
2225         c.Check(string(content), Equals, "mypassword")
2226         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2227         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2228         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2229         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2230 }
2231
2232 func (s *TestSuite) TestCalculateCost(c *C) {
2233         defer func(s string) { lockdir = s }(lockdir)
2234         lockdir = c.MkDir()
2235         now := time.Now()
2236         cr := s.runner
2237         cr.costStartTime = now.Add(-time.Hour)
2238         var logbuf bytes.Buffer
2239         cr.CrunchLog.Immediate = log.New(&logbuf, "", 0)
2240
2241         // if there's no InstanceType env var, cost is calculated as 0
2242         os.Unsetenv("InstanceType")
2243         cost := cr.calculateCost(now)
2244         c.Check(cost, Equals, 0.0)
2245
2246         // with InstanceType env var and loadPrices() hasn't run (or
2247         // hasn't found any data), cost is calculated based on
2248         // InstanceType env var
2249         os.Setenv("InstanceType", `{"Price":1.2}`)
2250         defer os.Unsetenv("InstanceType")
2251         cost = cr.calculateCost(now)
2252         c.Check(cost, Equals, 1.2)
2253
2254         // first update tells us the spot price was $1/h until 30
2255         // minutes ago when it increased to $2/h
2256         j, err := json.Marshal([]cloud.InstancePrice{
2257                 {StartTime: now.Add(-4 * time.Hour), Price: 1.0},
2258                 {StartTime: now.Add(-time.Hour / 2), Price: 2.0},
2259         })
2260         c.Assert(err, IsNil)
2261         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2262         cr.loadPrices()
2263         cost = cr.calculateCost(now)
2264         c.Check(cost, Equals, 1.5)
2265
2266         // next update (via --list + SIGUSR2) tells us the spot price
2267         // increased to $3/h 15 minutes ago
2268         j, err = json.Marshal([]cloud.InstancePrice{
2269                 {StartTime: now.Add(-time.Hour / 3), Price: 2.0}, // dup of -time.Hour/2 price
2270                 {StartTime: now.Add(-time.Hour / 4), Price: 3.0},
2271         })
2272         c.Assert(err, IsNil)
2273         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2274         cr.loadPrices()
2275         cost = cr.calculateCost(now)
2276         c.Check(cost, Equals, 1.0/2+2.0/4+3.0/4)
2277
2278         cost = cr.calculateCost(now.Add(-time.Hour / 2))
2279         c.Check(cost, Equals, 0.5)
2280
2281         c.Logf("%s", logbuf.String())
2282         c.Check(logbuf.String(), Matches, `(?ms).*Instance price changed to 1\.00 at 20.* changed to 2\.00 .* changed to 3\.00 .*`)
2283         c.Check(logbuf.String(), Not(Matches), `(?ms).*changed to 2\.00 .* changed to 2\.00 .*`)
2284 }
2285
2286 type FakeProcess struct {
2287         cmdLine []string
2288 }
2289
2290 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2291         return fp.cmdLine, nil
2292 }