lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "log"
  16         "math/rand"
  17         "net/http"
  18         "net/http/httptest"
  19         "os"
  20         "os/exec"
  21         "regexp"
  22         "runtime/pprof"
  23         "strconv"
  24         "strings"
  25         "sync"
  26         "sync/atomic"
  27         "syscall"
  28         "testing"
  29         "time"
  30
  31         "git.arvados.org/arvados.git/lib/cloud"
  32         "git.arvados.org/arvados.git/lib/cmd"
  33         "git.arvados.org/arvados.git/sdk/go/arvados"
  34         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  35         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  36         "git.arvados.org/arvados.git/sdk/go/manifest"
  37         "golang.org/x/net/context"
  38
  39         . "gopkg.in/check.v1"
  40 )
  41
  42 // Gocheck boilerplate
  43 func TestCrunchExec(t *testing.T) {
  44         TestingT(t)
  45 }
  46
  47 const logLineStart = `(?m)(.*\n)*\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z `
  48
  49 var _ = Suite(&TestSuite{})
  50
  51 type TestSuite struct {
  52         client                   *arvados.Client
  53         api                      *ArvTestClient
  54         runner                   *ContainerRunner
  55         executor                 *stubExecutor
  56         keepmount                string
  57         keepmountTmp             []string
  58         testDispatcherKeepClient KeepTestClient
  59         testContainerKeepClient  KeepTestClient
  60 }
  61
  62 func (s *TestSuite) SetUpTest(c *C) {
  63         s.client = arvados.NewClientFromEnv()
  64         s.executor = &stubExecutor{}
  65         var err error
  66         s.api = &ArvTestClient{}
  67         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  68         c.Assert(err, IsNil)
  69         s.runner.executor = s.executor
  70         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  71                 return s.api, &s.testContainerKeepClient, s.client, nil
  72         }
  73         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  74                 s.runner.ArvMountPoint = s.keepmount
  75                 for i, opt := range cmd {
  76                         if opt == "--mount-tmp" {
  77                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  78                                 if err != nil {
  79                                         return nil, err
  80                                 }
  81                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  82                         }
  83                 }
  84                 return nil, nil
  85         }
  86         s.keepmount = c.MkDir()
  87         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  88         s.keepmountTmp = nil
  89         c.Assert(err, IsNil)
  90         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  91         c.Assert(err, IsNil)
  92         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  93         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  94         c.Assert(err, IsNil)
  95         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  96         c.Assert(err, IsNil)
  97         s.runner.ArvMountPoint = s.keepmount
  98 }
  99
 100 type ArvTestClient struct {
 101         Total   int64
 102         Calls   int
 103         Content []arvadosclient.Dict
 104         arvados.Container
 105         secretMounts []byte
 106         Logs         map[string]*bytes.Buffer
 107         sync.Mutex
 108         WasSetRunning bool
 109         callraw       bool
 110 }
 111
 112 type KeepTestClient struct {
 113         Called         bool
 114         Content        []byte
 115         StorageClasses []string
 116 }
 117
 118 type stubExecutor struct {
 119         imageLoaded bool
 120         loaded      string
 121         loadErr     error
 122         exitCode    int
 123         createErr   error
 124         created     containerSpec
 125         startErr    error
 126         waitSleep   time.Duration
 127         waitErr     error
 128         stopErr     error
 129         stopped     bool
 130         closed      bool
 131         runFunc     func() int
 132         exit        chan int
 133 }
 134
 135 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 136         containerClient *arvados.Client) error {
 137         e.loaded = tarball
 138         return e.loadErr
 139 }
 140 func (e *stubExecutor) Runtime() string                 { return "stub" }
 141 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 142 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 143 func (e *stubExecutor) Start() error {
 144         e.exit = make(chan int, 1)
 145         go func() { e.exit <- e.runFunc() }()
 146         return e.startErr
 147 }
 148 func (e *stubExecutor) CgroupID() string { return "cgroupid" }
 149 func (e *stubExecutor) Stop() error      { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 150 func (e *stubExecutor) Close()           { e.closed = true }
 151 func (e *stubExecutor) Wait(context.Context) (int, error) {
 152         return <-e.exit, e.waitErr
 153 }
 154 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 155         return nil, errors.New("unimplemented")
 156 }
 157 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 158
 159 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 160
 161 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 162 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 163 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 164
 165 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 166 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 167
 168 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 169 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 170 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 171 `
 172
 173 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 174
 175 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 176 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 177
 178 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 179 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 180
 181 func (client *ArvTestClient) Create(resourceType string,
 182         parameters arvadosclient.Dict,
 183         output interface{}) error {
 184
 185         client.Mutex.Lock()
 186         defer client.Mutex.Unlock()
 187
 188         client.Calls++
 189         client.Content = append(client.Content, parameters)
 190
 191         if resourceType == "logs" {
 192                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 193                 if client.Logs == nil {
 194                         client.Logs = make(map[string]*bytes.Buffer)
 195                 }
 196                 if client.Logs[et] == nil {
 197                         client.Logs[et] = &bytes.Buffer{}
 198                 }
 199                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 200         }
 201
 202         if resourceType == "collections" && output != nil {
 203                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 204                 md5sum := md5.Sum([]byte(mt))
 205                 outmap := output.(*arvados.Collection)
 206                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 207                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 208         }
 209
 210         return nil
 211 }
 212
 213 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 214         switch {
 215         case method == "GET" && resourceType == "containers" && action == "auth":
 216                 return json.Unmarshal([]byte(`{
 217                         "kind": "arvados#api_client_authorization",
 218                         "uuid": "`+fakeAuthUUID+`",
 219                         "api_token": "`+fakeAuthToken+`"
 220                         }`), output)
 221         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 222                 if client.secretMounts != nil {
 223                         return json.Unmarshal(client.secretMounts, output)
 224                 }
 225                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 226         default:
 227                 return fmt.Errorf("Not found")
 228         }
 229 }
 230
 231 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 232         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 233         var j []byte
 234         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 235                 j = []byte(`{
 236                         "kind": "arvados#nodeList",
 237                         "items": [{
 238                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 239                                 "hostname": "compute2",
 240                                 "properties": {"total_cpu_cores": 16}
 241                         }]}`)
 242         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 243                 if uuid == "" {
 244                         j, err = json.Marshal(map[string]interface{}{
 245                                 "items": []interface{}{client.Container},
 246                                 "kind":  "arvados#nodeList",
 247                         })
 248                 } else {
 249                         j, err = json.Marshal(client.Container)
 250                 }
 251         } else {
 252                 j = []byte(`{
 253                         "command": ["sleep", "1"],
 254                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 255                         "cwd": ".",
 256                         "environment": {},
 257                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 258                         "output_path": "/tmp",
 259                         "priority": 1,
 260                         "runtime_constraints": {}
 261                 }`)
 262         }
 263         return ioutil.NopCloser(bytes.NewReader(j)), err
 264 }
 265
 266 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 267         if resourceType == "collections" {
 268                 if uuid == hwPDH {
 269                         output.(*arvados.Collection).ManifestText = hwManifest
 270                 } else if uuid == otherPDH {
 271                         output.(*arvados.Collection).ManifestText = otherManifest
 272                 } else if uuid == normalizedWithSubdirsPDH {
 273                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 274                 } else if uuid == denormalizedWithSubdirsPDH {
 275                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 276                 }
 277         }
 278         if resourceType == "containers" {
 279                 (*output.(*arvados.Container)) = client.Container
 280         }
 281         return nil
 282 }
 283
 284 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 285         client.Mutex.Lock()
 286         defer client.Mutex.Unlock()
 287         client.Calls++
 288         client.Content = append(client.Content, parameters)
 289         if resourceType == "containers" {
 290                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 291                         client.WasSetRunning = true
 292                 }
 293         } else if resourceType == "collections" && output != nil {
 294                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 295                 output.(*arvados.Collection).UUID = uuid
 296                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 297         }
 298         return nil
 299 }
 300
 301 var discoveryMap = map[string]interface{}{
 302         "defaultTrashLifetime":               float64(1209600),
 303         "crunchLimitLogBytesPerJob":          float64(67108864),
 304         "crunchLogThrottleBytes":             float64(65536),
 305         "crunchLogThrottlePeriod":            float64(60),
 306         "crunchLogThrottleLines":             float64(1024),
 307         "crunchLogPartialLineThrottlePeriod": float64(5),
 308         "crunchLogBytesPerEvent":             float64(4096),
 309         "crunchLogSecondsBetweenEvents":      float64(1),
 310 }
 311
 312 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 313         return discoveryMap[key], nil
 314 }
 315
 316 // CalledWith returns the parameters from the first API call whose
 317 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 318 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 319 // no call matches, it returns nil.
 320 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 321 call:
 322         for _, content := range client.Content {
 323                 var v interface{} = content
 324                 for _, k := range strings.Split(jpath, ".") {
 325                         if dict, ok := v.(arvadosclient.Dict); !ok {
 326                                 continue call
 327                         } else {
 328                                 v = dict[k]
 329                         }
 330                 }
 331                 if v == expect {
 332                         return content
 333                 }
 334         }
 335         return nil
 336 }
 337
 338 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 339         return locator, nil
 340 }
 341
 342 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 343         client.Content = opts.Data
 344         return arvados.BlockWriteResponse{
 345                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 346         }, nil
 347 }
 348
 349 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 350         return 0, errors.New("not implemented")
 351 }
 352
 353 func (client *KeepTestClient) ClearBlockCache() {
 354 }
 355
 356 func (client *KeepTestClient) Close() {
 357         client.Content = nil
 358 }
 359
 360 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 361         client.StorageClasses = sc
 362 }
 363
 364 type FileWrapper struct {
 365         io.ReadCloser
 366         len int64
 367 }
 368
 369 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 370         return nil, errors.New("not implemented")
 371 }
 372
 373 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 374         return 0, errors.New("not implemented")
 375 }
 376
 377 func (fw FileWrapper) Size() int64 {
 378         return fw.len
 379 }
 380
 381 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 382         return nil, errors.New("not implemented")
 383 }
 384
 385 func (fw FileWrapper) Truncate(int64) error {
 386         return errors.New("not implemented")
 387 }
 388
 389 func (fw FileWrapper) Write([]byte) (int, error) {
 390         return 0, errors.New("not implemented")
 391 }
 392
 393 func (fw FileWrapper) Sync() error {
 394         return errors.New("not implemented")
 395 }
 396
 397 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 398         return nil, errors.New("not implemented")
 399 }
 400
 401 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 402         return errors.New("not implemented")
 403 }
 404
 405 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 406         if filename == hwImageID+".tar" {
 407                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 408                 client.Called = true
 409                 return FileWrapper{rdr, 1321984}, nil
 410         } else if filename == "/file1_in_main.txt" {
 411                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 412                 client.Called = true
 413                 return FileWrapper{rdr, 3}, nil
 414         }
 415         return nil, nil
 416 }
 417
 418 func (s *TestSuite) TestLoadImage(c *C) {
 419         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 420         s.runner.Container.Mounts = map[string]arvados.Mount{
 421                 "/out": {Kind: "tmp", Writable: true},
 422         }
 423         s.runner.Container.OutputPath = "/out"
 424
 425         _, err := s.runner.SetupMounts()
 426         c.Assert(err, IsNil)
 427
 428         imageID, err := s.runner.LoadImage()
 429         c.Check(err, IsNil)
 430         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 431         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 432
 433         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 434         s.executor.imageLoaded = false
 435         s.executor.loaded = ""
 436         s.executor.loadErr = errors.New("bork")
 437         imageID, err = s.runner.LoadImage()
 438         c.Check(err, ErrorMatches, ".*bork")
 439         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 440
 441         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 442         s.executor.imageLoaded = false
 443         s.executor.loaded = ""
 444         s.executor.loadErr = nil
 445         imageID, err = s.runner.LoadImage()
 446         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 447         c.Check(s.executor.loaded, Equals, "")
 448 }
 449
 450 type ArvErrorTestClient struct{}
 451
 452 func (ArvErrorTestClient) Create(resourceType string,
 453         parameters arvadosclient.Dict,
 454         output interface{}) error {
 455         return nil
 456 }
 457
 458 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 459         if method == "GET" && resourceType == "containers" && action == "auth" {
 460                 return nil
 461         }
 462         return errors.New("ArvError")
 463 }
 464
 465 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 466         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 467         return nil, errors.New("ArvError")
 468 }
 469
 470 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 471         return errors.New("ArvError")
 472 }
 473
 474 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 475         return nil
 476 }
 477
 478 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 479         return discoveryMap[key], nil
 480 }
 481
 482 type KeepErrorTestClient struct {
 483         KeepTestClient
 484 }
 485
 486 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 487         return nil, errors.New("KeepError")
 488 }
 489
 490 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 491         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 492 }
 493
 494 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 495         return "", errors.New("KeepError")
 496 }
 497
 498 type KeepReadErrorTestClient struct {
 499         KeepTestClient
 500 }
 501
 502 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 503         return 0, errors.New("KeepError")
 504 }
 505
 506 type ErrorReader struct {
 507         FileWrapper
 508 }
 509
 510 func (ErrorReader) Read(p []byte) (n int, err error) {
 511         return 0, errors.New("ErrorReader")
 512 }
 513
 514 func (ErrorReader) Seek(int64, int) (int64, error) {
 515         return 0, errors.New("ErrorReader")
 516 }
 517
 518 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 519         return ErrorReader{}, nil
 520 }
 521
 522 type ClosableBuffer struct {
 523         bytes.Buffer
 524 }
 525
 526 func (*ClosableBuffer) Close() error {
 527         return nil
 528 }
 529
 530 type TestLogs struct {
 531         Stdout ClosableBuffer
 532         Stderr ClosableBuffer
 533 }
 534
 535 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 536         if logstr == "stdout" {
 537                 return &tl.Stdout, nil
 538         }
 539         if logstr == "stderr" {
 540                 return &tl.Stderr, nil
 541         }
 542         return nil, errors.New("???")
 543 }
 544
 545 func dockerLog(fd byte, msg string) []byte {
 546         by := []byte(msg)
 547         header := make([]byte, 8+len(by))
 548         header[0] = fd
 549         header[7] = byte(len(by))
 550         copy(header[8:], by)
 551         return header
 552 }
 553
 554 func (s *TestSuite) TestRunContainer(c *C) {
 555         s.executor.runFunc = func() int {
 556                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 557                 return 0
 558         }
 559
 560         var logs TestLogs
 561         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 562         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 563         s.runner.Container.Command = []string{"./hw"}
 564         s.runner.Container.OutputStorageClasses = []string{"default"}
 565
 566         imageID, err := s.runner.LoadImage()
 567         c.Assert(err, IsNil)
 568
 569         err = s.runner.CreateContainer(imageID, nil)
 570         c.Assert(err, IsNil)
 571
 572         err = s.runner.StartContainer()
 573         c.Assert(err, IsNil)
 574
 575         err = s.runner.WaitFinish()
 576         c.Assert(err, IsNil)
 577
 578         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 579         c.Check(logs.Stderr.String(), Equals, "")
 580 }
 581
 582 func (s *TestSuite) TestCommitLogs(c *C) {
 583         api := &ArvTestClient{}
 584         kc := &KeepTestClient{}
 585         defer kc.Close()
 586         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 587         c.Assert(err, IsNil)
 588         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 589
 590         cr.CrunchLog.Print("Hello world!")
 591         cr.CrunchLog.Print("Goodbye")
 592         cr.finalState = "Complete"
 593
 594         err = cr.CommitLogs()
 595         c.Check(err, IsNil)
 596
 597         c.Check(api.Calls, Equals, 2)
 598         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 599         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 600         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 601         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 602 }
 603
 604 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 605         api := &ArvTestClient{}
 606         kc := &KeepTestClient{}
 607         defer kc.Close()
 608         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 609         c.Assert(err, IsNil)
 610
 611         err = cr.UpdateContainerRunning("")
 612         c.Check(err, IsNil)
 613
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 615 }
 616
 617 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 618         api := &ArvTestClient{}
 619         kc := &KeepTestClient{}
 620         defer kc.Close()
 621         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 622         c.Assert(err, IsNil)
 623
 624         cr.LogsPDH = new(string)
 625         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 626
 627         cr.ExitCode = new(int)
 628         *cr.ExitCode = 42
 629         cr.finalState = "Complete"
 630
 631         err = cr.UpdateContainerFinal()
 632         c.Check(err, IsNil)
 633
 634         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 635         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 636         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 637 }
 638
 639 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 640         api := &ArvTestClient{}
 641         kc := &KeepTestClient{}
 642         defer kc.Close()
 643         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 644         c.Assert(err, IsNil)
 645         cr.cCancelled = true
 646         cr.finalState = "Cancelled"
 647
 648         err = cr.UpdateContainerFinal()
 649         c.Check(err, IsNil)
 650
 651         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 652         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 653         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 654 }
 655
 656 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 657 // dress rehearsal of the Run() function, starting from a JSON container record.
 658 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, fn func() int) (*ArvTestClient, *ContainerRunner, string) {
 659         err := json.Unmarshal([]byte(record), &s.api.Container)
 660         c.Assert(err, IsNil)
 661         initialState := s.api.Container.State
 662
 663         var sm struct {
 664                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 665         }
 666         err = json.Unmarshal([]byte(record), &sm)
 667         c.Check(err, IsNil)
 668         secretMounts, err := json.Marshal(sm)
 669         c.Assert(err, IsNil)
 670         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 671
 672         s.executor.runFunc = fn
 673
 674         s.runner.statInterval = 100 * time.Millisecond
 675         s.runner.containerWatchdogInterval = time.Second
 676
 677         realTemp := c.MkDir()
 678         tempcount := 0
 679         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 680                 tempcount++
 681                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 682                 err := os.Mkdir(d, os.ModePerm)
 683                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 684                         // Test case must have pre-populated the tempdir
 685                         err = nil
 686                 }
 687                 return d, err
 688         }
 689         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 690                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 691         }
 692
 693         if extraMounts != nil && len(extraMounts) > 0 {
 694                 err := s.runner.SetupArvMountPoint("keep")
 695                 c.Check(err, IsNil)
 696
 697                 for _, m := range extraMounts {
 698                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 699                 }
 700         }
 701
 702         err = s.runner.Run()
 703         if s.api.CalledWith("container.state", "Complete") != nil {
 704                 c.Check(err, IsNil)
 705         }
 706         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 707                 c.Check(s.api.WasSetRunning, Equals, true)
 708                 var lastupdate arvadosclient.Dict
 709                 for _, content := range s.api.Content {
 710                         if content["container"] != nil {
 711                                 lastupdate = content["container"].(arvadosclient.Dict)
 712                         }
 713                 }
 714                 if lastupdate["log"] == nil {
 715                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 716                 }
 717         }
 718
 719         if err != nil {
 720                 for k, v := range s.api.Logs {
 721                         c.Log(k)
 722                         c.Log(v.String())
 723                 }
 724         }
 725
 726         return s.api, s.runner, realTemp
 727 }
 728
 729 func (s *TestSuite) TestFullRunHello(c *C) {
 730         s.runner.enableMemoryLimit = true
 731         s.runner.networkMode = "default"
 732         s.fullRunHelper(c, `{
 733     "command": ["echo", "hello world"],
 734     "container_image": "`+arvadostest.DockerImage112PDH+`",
 735     "cwd": ".",
 736     "environment": {"foo":"bar","baz":"waz"},
 737     "mounts": {"/tmp": {"kind": "tmp"} },
 738     "output_path": "/tmp",
 739     "priority": 1,
 740     "runtime_constraints": {"vcpus":1,"ram":1000000},
 741     "state": "Locked",
 742     "output_storage_classes": ["default"]
 743 }`, nil, func() int {
 744                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 745                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 746                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 747                 c.Check(s.executor.created.VCPUs, Equals, 1)
 748                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 749                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 750                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 751                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 752                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 753                 return 0
 754         })
 755
 756         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 757         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 758         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 759         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 760         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 761 }
 762
 763 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 764         var ran bool
 765         s.fullRunHelper(c, `{
 766     "command": ["sleep", "3"],
 767     "container_image": "`+arvadostest.DockerImage112PDH+`",
 768     "cwd": ".",
 769     "environment": {},
 770     "mounts": {"/tmp": {"kind": "tmp"} },
 771     "output_path": "/tmp",
 772     "priority": 1,
 773     "runtime_constraints": {},
 774     "scheduling_parameters":{"max_run_time": 1},
 775     "state": "Running"
 776 }`, nil, func() int {
 777                 ran = true
 778                 return 2
 779         })
 780         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 781         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 782         c.Check(ran, Equals, false)
 783 }
 784
 785 func ec2MetadataServerStub(c *C, token *string, failureRate float64, stoptime *atomic.Value) *httptest.Server {
 786         failedOnce := false
 787         return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 788                 if !failedOnce || rand.Float64() < failureRate {
 789                         w.WriteHeader(http.StatusServiceUnavailable)
 790                         failedOnce = true
 791                         return
 792                 }
 793                 switch r.URL.Path {
 794                 case "/latest/api/token":
 795                         fmt.Fprintln(w, *token)
 796                 case "/latest/meta-data/spot/instance-action":
 797                         if r.Header.Get("X-aws-ec2-metadata-token") != *token {
 798                                 w.WriteHeader(http.StatusUnauthorized)
 799                         } else if t, _ := stoptime.Load().(time.Time); t.IsZero() {
 800                                 w.WriteHeader(http.StatusNotFound)
 801                         } else {
 802                                 fmt.Fprintf(w, `{"action":"stop","time":"%s"}`, t.Format(time.RFC3339))
 803                         }
 804                 default:
 805                         w.WriteHeader(http.StatusNotFound)
 806                 }
 807         }))
 808 }
 809
 810 func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
 811         s.testSpotInterruptionNotice(c, 0.1)
 812 }
 813
 814 func (s *TestSuite) TestSpotInterruptionNoticeNotAvailable(c *C) {
 815         s.testSpotInterruptionNotice(c, 1)
 816 }
 817
 818 func (s *TestSuite) testSpotInterruptionNotice(c *C, failureRate float64) {
 819         var stoptime atomic.Value
 820         token := "fake-ec2-metadata-token"
 821         stub := ec2MetadataServerStub(c, &token, failureRate, &stoptime)
 822         defer stub.Close()
 823
 824         defer func(i time.Duration, u string) {
 825                 spotInterruptionCheckInterval = i
 826                 ec2MetadataBaseURL = u
 827         }(spotInterruptionCheckInterval, ec2MetadataBaseURL)
 828         spotInterruptionCheckInterval = time.Second / 8
 829         ec2MetadataBaseURL = stub.URL
 830
 831         go s.runner.checkSpotInterruptionNotices()
 832         s.fullRunHelper(c, `{
 833     "command": ["sleep", "3"],
 834     "container_image": "`+arvadostest.DockerImage112PDH+`",
 835     "cwd": ".",
 836     "environment": {},
 837     "mounts": {"/tmp": {"kind": "tmp"} },
 838     "output_path": "/tmp",
 839     "priority": 1,
 840     "runtime_constraints": {},
 841     "state": "Locked"
 842 }`, nil, func() int {
 843                 time.Sleep(time.Second)
 844                 stoptime.Store(time.Now().Add(time.Minute).UTC())
 845                 token = "different-fake-ec2-metadata-token"
 846                 time.Sleep(time.Second)
 847                 return 0
 848         })
 849         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Checking for spot interruptions every 125ms using instance metadata at http://.*`)
 850         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Error checking spot interruptions: 503 Service Unavailable.*`)
 851         if failureRate == 1 {
 852                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Giving up on checking spot interruptions after too many consecutive failures.*`)
 853         } else {
 854                 text := `Cloud provider scheduled instance stop at ` + stoptime.Load().(time.Time).Format(time.RFC3339)
 855                 c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*`+text+`.*`)
 856                 c.Check(s.api.CalledWith("container.runtime_status.warning", "preemption notice"), NotNil)
 857                 c.Check(s.api.CalledWith("container.runtime_status.warningDetail", text), NotNil)
 858                 c.Check(s.api.CalledWith("container.runtime_status.preemptionNotice", text), NotNil)
 859         }
 860 }
 861
 862 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 863         s.fullRunHelper(c, `{
 864     "command": ["sleep", "3"],
 865     "container_image": "`+arvadostest.DockerImage112PDH+`",
 866     "cwd": ".",
 867     "environment": {},
 868     "mounts": {"/tmp": {"kind": "tmp"} },
 869     "output_path": "/tmp",
 870     "priority": 1,
 871     "runtime_constraints": {},
 872     "scheduling_parameters":{"max_run_time": 1},
 873     "state": "Locked"
 874 }`, nil, func() int {
 875                 time.Sleep(3 * time.Second)
 876                 return 0
 877         })
 878
 879         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 880         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 881 }
 882
 883 func (s *TestSuite) TestContainerWaitFails(c *C) {
 884         s.fullRunHelper(c, `{
 885     "command": ["sleep", "3"],
 886     "container_image": "`+arvadostest.DockerImage112PDH+`",
 887     "cwd": ".",
 888     "mounts": {"/tmp": {"kind": "tmp"} },
 889     "output_path": "/tmp",
 890     "priority": 1,
 891     "state": "Locked"
 892 }`, nil, func() int {
 893                 s.executor.waitErr = errors.New("Container is not running")
 894                 return 0
 895         })
 896
 897         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 898         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 899 }
 900
 901 func (s *TestSuite) TestCrunchstat(c *C) {
 902         s.fullRunHelper(c, `{
 903                 "command": ["sleep", "1"],
 904                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 905                 "cwd": ".",
 906                 "environment": {},
 907                 "mounts": {"/tmp": {"kind": "tmp"} },
 908                 "output_path": "/tmp",
 909                 "priority": 1,
 910                 "runtime_constraints": {},
 911                 "state": "Locked"
 912         }`, nil, func() int {
 913                 time.Sleep(time.Second)
 914                 return 0
 915         })
 916
 917         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 918         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 919
 920         // We didn't actually start a container, so crunchstat didn't
 921         // find accounting files and therefore didn't log any stats.
 922         // It should have logged a "can't find accounting files"
 923         // message after one poll interval, though, so we can confirm
 924         // it's alive:
 925         c.Assert(s.api.Logs["crunchstat"], NotNil)
 926         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 927
 928         // The "files never appeared" log assures us that we called
 929         // (*crunchstat.Reporter)Stop(), and that we set it up with
 930         // the correct container ID "abcde":
 931         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 932 }
 933
 934 func (s *TestSuite) TestNodeInfoLog(c *C) {
 935         os.Setenv("SLURMD_NODENAME", "compute2")
 936         s.fullRunHelper(c, `{
 937                 "command": ["sleep", "1"],
 938                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 939                 "cwd": ".",
 940                 "environment": {},
 941                 "mounts": {"/tmp": {"kind": "tmp"} },
 942                 "output_path": "/tmp",
 943                 "priority": 1,
 944                 "runtime_constraints": {},
 945                 "state": "Locked"
 946         }`, nil, func() int {
 947                 time.Sleep(time.Second)
 948                 return 0
 949         })
 950
 951         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 952         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 953
 954         c.Assert(s.api.Logs["node"], NotNil)
 955         json := s.api.Logs["node"].String()
 956         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 957         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 958         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 959
 960         c.Assert(s.api.Logs["node-info"], NotNil)
 961         json = s.api.Logs["node-info"].String()
 962         c.Check(json, Matches, `(?ms).*Host Information.*`)
 963         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 964         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 965         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 966         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 967 }
 968
 969 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 970         s.fullRunHelper(c, `{
 971                 "command": ["sleep", "1"],
 972                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 973                 "cwd": ".",
 974                 "environment": {},
 975                 "mounts": {"/tmp": {"kind": "tmp"} },
 976                 "output_path": "/tmp",
 977                 "priority": 1,
 978                 "runtime_constraints": {},
 979                 "state": "Locked"
 980         }`, nil, func() int {
 981                 return 0
 982         })
 983
 984         c.Assert(s.api.Logs["crunch-run"], NotNil)
 985         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 986         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 987         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 988         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 989 }
 990
 991 func (s *TestSuite) testLogRSSThresholds(c *C, ram int, expected []int, notExpected int) {
 992         s.runner.cgroupRoot = "testdata/fakestat"
 993         s.fullRunHelper(c, `{
 994                 "command": ["true"],
 995                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 996                 "cwd": ".",
 997                 "environment": {},
 998                 "mounts": {"/tmp": {"kind": "tmp"} },
 999                 "output_path": "/tmp",
1000                 "priority": 1,
1001                 "runtime_constraints": {"ram": `+strconv.Itoa(ram)+`},
1002                 "state": "Locked"
1003         }`, nil, func() int { return 0 })
1004         logs := s.api.Logs["crunch-run"].String()
1005         pattern := logLineStart + `Container using over %d%% of memory \(rss 734003200/%d bytes\)`
1006         var threshold int
1007         for _, threshold = range expected {
1008                 c.Check(logs, Matches, fmt.Sprintf(pattern, threshold, ram))
1009         }
1010         if notExpected > threshold {
1011                 c.Check(logs, Not(Matches), fmt.Sprintf(pattern, notExpected, ram))
1012         }
1013 }
1014
1015 func (s *TestSuite) TestLogNoRSSThresholds(c *C) {
1016         s.testLogRSSThresholds(c, 7340032000, []int{}, 90)
1017 }
1018
1019 func (s *TestSuite) TestLogSomeRSSThresholds(c *C) {
1020         onePercentRSS := 7340032
1021         s.testLogRSSThresholds(c, 102*onePercentRSS, []int{90, 95}, 99)
1022 }
1023
1024 func (s *TestSuite) TestLogAllRSSThresholds(c *C) {
1025         s.testLogRSSThresholds(c, 734003299, []int{90, 95, 99}, 0)
1026 }
1027
1028 func (s *TestSuite) TestLogMaximaAfterRun(c *C) {
1029         s.runner.cgroupRoot = "testdata/fakestat"
1030         s.runner.parentTemp = c.MkDir()
1031         s.fullRunHelper(c, `{
1032         "command": ["true"],
1033         "container_image": "`+arvadostest.DockerImage112PDH+`",
1034         "cwd": ".",
1035         "environment": {},
1036         "mounts": {"/tmp": {"kind": "tmp"} },
1037         "output_path": "/tmp",
1038         "priority": 1,
1039         "runtime_constraints": {"ram": 7340032000},
1040         "state": "Locked"
1041     }`, nil, func() int { return 0 })
1042         logs := s.api.Logs["crunch-run"].String()
1043         for _, expected := range []string{
1044                 `Maximum disk usage was \d+%, \d+/\d+ bytes`,
1045                 `Maximum container memory cache usage was 73400320 bytes`,
1046                 `Maximum container memory swap usage was 320 bytes`,
1047                 `Maximum container memory pgmajfault usage was 20 faults`,
1048                 `Maximum container memory rss usage was 10%, 734003200/7340032000 bytes`,
1049                 `Maximum crunch-run memory rss usage was \d+ bytes`,
1050         } {
1051                 c.Check(logs, Matches, logLineStart+expected)
1052         }
1053 }
1054
1055 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
1056         var collection_create, container_update arvadosclient.Dict
1057         s.fullRunHelper(c, `{
1058                 "command": ["true"],
1059                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1060                 "cwd": ".",
1061                 "environment": {},
1062                 "mounts": {"/tmp": {"kind": "tmp"} },
1063                 "output_path": "/tmp",
1064                 "priority": 1,
1065                 "runtime_constraints": {},
1066                 "state": "Locked",
1067                 "uuid": "zzzzz-dz642-202301121543210"
1068         }`, nil, func() int {
1069                 collection_create = s.api.CalledWith("ensure_unique_name", true)
1070                 container_update = s.api.CalledWith("container.state", "Running")
1071                 return 0
1072         })
1073
1074         c.Assert(collection_create, NotNil)
1075         log_collection := collection_create["collection"].(arvadosclient.Dict)
1076         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
1077         manifest_text := log_collection["manifest_text"].(string)
1078         // We check that the file size is at least two digits as an easy way to
1079         // check the file isn't empty.
1080         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node-info\.txt( .+)?\n`)
1081         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node\.json( .+)?\n`)
1082
1083         c.Assert(container_update, NotNil)
1084         // As of Arvados 2.5.0, the container update must specify its log in PDH
1085         // format for the API server to propagate it to container requests, which
1086         // is what we care about for this test.
1087         expect_pdh := fmt.Sprintf("%x+%d", md5.Sum([]byte(manifest_text)), len(manifest_text))
1088         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Equals, expect_pdh)
1089 }
1090
1091 func (s *TestSuite) TestContainerRecordLog(c *C) {
1092         s.fullRunHelper(c, `{
1093                 "command": ["sleep", "1"],
1094                 "container_image": "`+arvadostest.DockerImage112PDH+`",
1095                 "cwd": ".",
1096                 "environment": {},
1097                 "mounts": {"/tmp": {"kind": "tmp"} },
1098                 "output_path": "/tmp",
1099                 "priority": 1,
1100                 "runtime_constraints": {},
1101                 "state": "Locked"
1102         }`, nil,
1103                 func() int {
1104                         time.Sleep(time.Second)
1105                         return 0
1106                 })
1107
1108         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1109         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1110
1111         c.Assert(s.api.Logs["container"], NotNil)
1112         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
1113 }
1114
1115 func (s *TestSuite) TestFullRunStderr(c *C) {
1116         s.fullRunHelper(c, `{
1117     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
1118     "container_image": "`+arvadostest.DockerImage112PDH+`",
1119     "cwd": ".",
1120     "environment": {},
1121     "mounts": {"/tmp": {"kind": "tmp"} },
1122     "output_path": "/tmp",
1123     "priority": 1,
1124     "runtime_constraints": {},
1125     "state": "Locked"
1126 }`, nil, func() int {
1127                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1128                 fmt.Fprintln(s.executor.created.Stderr, "world")
1129                 return 1
1130         })
1131
1132         final := s.api.CalledWith("container.state", "Complete")
1133         c.Assert(final, NotNil)
1134         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1135         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1136
1137         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
1138         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
1139 }
1140
1141 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
1142         s.fullRunHelper(c, `{
1143     "command": ["pwd"],
1144     "container_image": "`+arvadostest.DockerImage112PDH+`",
1145     "cwd": ".",
1146     "environment": {},
1147     "mounts": {"/tmp": {"kind": "tmp"} },
1148     "output_path": "/tmp",
1149     "priority": 1,
1150     "runtime_constraints": {},
1151     "state": "Locked"
1152 }`, nil, func() int {
1153                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
1154                 return 0
1155         })
1156
1157         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1158         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1159         c.Log(s.api.Logs["stdout"])
1160         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
1161 }
1162
1163 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1164         s.fullRunHelper(c, `{
1165     "command": ["pwd"],
1166     "container_image": "`+arvadostest.DockerImage112PDH+`",
1167     "cwd": "/bin",
1168     "environment": {},
1169     "mounts": {"/tmp": {"kind": "tmp"} },
1170     "output_path": "/tmp",
1171     "priority": 1,
1172     "runtime_constraints": {},
1173     "state": "Locked"
1174 }`, nil, func() int {
1175                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1176                 return 0
1177         })
1178
1179         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1180         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1181         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1182 }
1183
1184 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1185         s.fullRunHelper(c, `{
1186     "command": ["pwd"],
1187     "container_image": "`+arvadostest.DockerImage112PDH+`",
1188     "cwd": "/bin",
1189     "environment": {},
1190     "mounts": {"/tmp": {"kind": "tmp"} },
1191     "output_path": "/tmp",
1192     "priority": 1,
1193     "runtime_constraints": {},
1194     "state": "Locked",
1195     "output_storage_classes": ["foo", "bar"]
1196 }`, nil, func() int {
1197                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1198                 return 0
1199         })
1200
1201         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1202         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1203         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1204         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1205         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1206 }
1207
1208 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1209         s.fullRunHelper(c, `{
1210     "command": ["pwd"],
1211     "container_image": "`+arvadostest.DockerImage112PDH+`",
1212     "cwd": "/bin",
1213     "environment": {},
1214     "mounts": {"/tmp": {"kind": "tmp"} },
1215     "output_path": "/tmp",
1216     "priority": 1,
1217     "runtime_constraints": {"cuda": {"device_count": 2}},
1218     "state": "Locked",
1219     "output_storage_classes": ["foo", "bar"]
1220 }`, nil, func() int {
1221                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1222                 return 0
1223         })
1224         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1225 }
1226
1227 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1228         s.fullRunHelper(c, `{
1229     "command": ["pwd"],
1230     "container_image": "`+arvadostest.DockerImage112PDH+`",
1231     "cwd": "/bin",
1232     "environment": {},
1233     "mounts": {"/tmp": {"kind": "tmp"} },
1234     "output_path": "/tmp",
1235     "priority": 1,
1236     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1237     "state": "Locked",
1238     "output_storage_classes": ["foo", "bar"]
1239 }`, nil, func() int {
1240                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1241                 return 0
1242         })
1243         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1244 }
1245
1246 func (s *TestSuite) TestStopOnSignal(c *C) {
1247         s.executor.runFunc = func() int {
1248                 s.executor.created.Stdout.Write([]byte("foo\n"))
1249                 s.runner.SigChan <- syscall.SIGINT
1250                 time.Sleep(10 * time.Second)
1251                 return 0
1252         }
1253         s.testStopContainer(c)
1254 }
1255
1256 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1257         s.executor.runFunc = func() int {
1258                 s.executor.created.Stdout.Write([]byte("foo\n"))
1259                 s.runner.ArvMountExit <- nil
1260                 close(s.runner.ArvMountExit)
1261                 time.Sleep(10 * time.Second)
1262                 return 0
1263         }
1264         s.runner.ArvMountExit = make(chan error)
1265         s.testStopContainer(c)
1266 }
1267
1268 func (s *TestSuite) testStopContainer(c *C) {
1269         record := `{
1270     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1271     "container_image": "` + arvadostest.DockerImage112PDH + `",
1272     "cwd": ".",
1273     "environment": {},
1274     "mounts": {"/tmp": {"kind": "tmp"} },
1275     "output_path": "/tmp",
1276     "priority": 1,
1277     "runtime_constraints": {},
1278     "state": "Locked"
1279 }`
1280
1281         err := json.Unmarshal([]byte(record), &s.api.Container)
1282         c.Assert(err, IsNil)
1283
1284         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1285         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1286                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1287         }
1288
1289         done := make(chan error)
1290         go func() {
1291                 done <- s.runner.Run()
1292         }()
1293         select {
1294         case <-time.After(20 * time.Second):
1295                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1296                 c.Fatal("timed out")
1297         case err = <-done:
1298                 c.Check(err, IsNil)
1299         }
1300         for k, v := range s.api.Logs {
1301                 c.Log(k)
1302                 c.Log(v.String(), "\n")
1303         }
1304
1305         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1306         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1307         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1308 }
1309
1310 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1311         s.fullRunHelper(c, `{
1312     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1313     "container_image": "`+arvadostest.DockerImage112PDH+`",
1314     "cwd": "/bin",
1315     "environment": {"FROBIZ": "bilbo"},
1316     "mounts": {"/tmp": {"kind": "tmp"} },
1317     "output_path": "/tmp",
1318     "priority": 1,
1319     "runtime_constraints": {},
1320     "state": "Locked"
1321 }`, nil, func() int {
1322                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1323                 return 0
1324         })
1325
1326         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1327         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1328         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1329 }
1330
1331 type ArvMountCmdLine struct {
1332         Cmd   []string
1333         token string
1334 }
1335
1336 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1337         am.Cmd = c
1338         am.token = token
1339         return nil, nil
1340 }
1341
1342 func stubCert(temp string) string {
1343         path := temp + "/ca-certificates.crt"
1344         crt, _ := os.Create(path)
1345         crt.Close()
1346         arvadosclient.CertFiles = []string{path}
1347         return path
1348 }
1349
1350 func (s *TestSuite) TestSetupMounts(c *C) {
1351         cr := s.runner
1352         am := &ArvMountCmdLine{}
1353         cr.RunArvMount = am.ArvMountTest
1354         cr.ContainerArvClient = &ArvTestClient{}
1355         cr.ContainerKeepClient = &KeepTestClient{}
1356         cr.Container.OutputStorageClasses = []string{"default"}
1357
1358         realTemp := c.MkDir()
1359         certTemp := c.MkDir()
1360         stubCertPath := stubCert(certTemp)
1361         cr.parentTemp = realTemp
1362
1363         i := 0
1364         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1365                 i++
1366                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1367                 err := os.Mkdir(d, os.ModePerm)
1368                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1369                         // Test case must have pre-populated the tempdir
1370                         err = nil
1371                 }
1372                 return d, err
1373         }
1374
1375         checkEmpty := func() {
1376                 // Should be deleted.
1377                 _, err := os.Stat(realTemp)
1378                 c.Assert(os.IsNotExist(err), Equals, true)
1379
1380                 // Now recreate it for the next test.
1381                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1382         }
1383
1384         {
1385                 i = 0
1386                 cr.ArvMountPoint = ""
1387                 cr.Container.Mounts = make(map[string]arvados.Mount)
1388                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1389                 cr.Container.OutputPath = "/tmp"
1390                 cr.statInterval = 5 * time.Second
1391                 bindmounts, err := cr.SetupMounts()
1392                 c.Check(err, IsNil)
1393                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1394                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1395                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1396                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1397                 os.RemoveAll(cr.ArvMountPoint)
1398                 cr.CleanupDirs()
1399                 checkEmpty()
1400         }
1401
1402         {
1403                 i = 0
1404                 cr.ArvMountPoint = ""
1405                 cr.Container.Mounts = make(map[string]arvados.Mount)
1406                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1407                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1408                 cr.Container.OutputPath = "/out"
1409                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1410
1411                 bindmounts, err := cr.SetupMounts()
1412                 c.Check(err, IsNil)
1413                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1414                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1415                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1416                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1417                 os.RemoveAll(cr.ArvMountPoint)
1418                 cr.CleanupDirs()
1419                 checkEmpty()
1420         }
1421
1422         {
1423                 i = 0
1424                 cr.ArvMountPoint = ""
1425                 cr.Container.Mounts = make(map[string]arvados.Mount)
1426                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1427                 cr.Container.OutputPath = "/tmp"
1428                 cr.Container.RuntimeConstraints.API = true
1429                 cr.Container.OutputStorageClasses = []string{"default"}
1430
1431                 bindmounts, err := cr.SetupMounts()
1432                 c.Check(err, IsNil)
1433                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1434                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1435                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1436                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1437                 os.RemoveAll(cr.ArvMountPoint)
1438                 cr.CleanupDirs()
1439                 checkEmpty()
1440
1441                 cr.Container.RuntimeConstraints.API = false
1442         }
1443
1444         {
1445                 i = 0
1446                 cr.ArvMountPoint = ""
1447                 cr.Container.Mounts = map[string]arvados.Mount{
1448                         "/keeptmp": {Kind: "collection", Writable: true},
1449                 }
1450                 cr.Container.OutputPath = "/keeptmp"
1451
1452                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1453
1454                 bindmounts, err := cr.SetupMounts()
1455                 c.Check(err, IsNil)
1456                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1457                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1458                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1459                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1460                 os.RemoveAll(cr.ArvMountPoint)
1461                 cr.CleanupDirs()
1462                 checkEmpty()
1463         }
1464
1465         {
1466                 i = 0
1467                 cr.ArvMountPoint = ""
1468                 cr.Container.Mounts = map[string]arvados.Mount{
1469                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1470                         "/keepout": {Kind: "collection", Writable: true},
1471                 }
1472                 cr.Container.OutputPath = "/keepout"
1473
1474                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1475                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1476
1477                 bindmounts, err := cr.SetupMounts()
1478                 c.Check(err, IsNil)
1479                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1480                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1481                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1482                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1483                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1484                         "/keepout": {realTemp + "/keep1/tmp0", false},
1485                 })
1486                 os.RemoveAll(cr.ArvMountPoint)
1487                 cr.CleanupDirs()
1488                 checkEmpty()
1489         }
1490
1491         {
1492                 i = 0
1493                 cr.ArvMountPoint = ""
1494                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1495                 cr.Container.Mounts = map[string]arvados.Mount{
1496                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1497                         "/keepout": {Kind: "collection", Writable: true},
1498                 }
1499                 cr.Container.OutputPath = "/keepout"
1500
1501                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1502                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1503
1504                 bindmounts, err := cr.SetupMounts()
1505                 c.Check(err, IsNil)
1506                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1507                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1508                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1509                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1510                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1511                         "/keepout": {realTemp + "/keep1/tmp0", false},
1512                 })
1513                 os.RemoveAll(cr.ArvMountPoint)
1514                 cr.CleanupDirs()
1515                 checkEmpty()
1516         }
1517
1518         for _, test := range []struct {
1519                 in  interface{}
1520                 out string
1521         }{
1522                 {in: "foo", out: `"foo"`},
1523                 {in: nil, out: `null`},
1524                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1525         } {
1526                 i = 0
1527                 cr.ArvMountPoint = ""
1528                 cr.Container.Mounts = map[string]arvados.Mount{
1529                         "/mnt/test.json": {Kind: "json", Content: test.in},
1530                 }
1531                 bindmounts, err := cr.SetupMounts()
1532                 c.Check(err, IsNil)
1533                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1534                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1535                 })
1536                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1537                 c.Check(err, IsNil)
1538                 c.Check(content, DeepEquals, []byte(test.out))
1539                 os.RemoveAll(cr.ArvMountPoint)
1540                 cr.CleanupDirs()
1541                 checkEmpty()
1542         }
1543
1544         for _, test := range []struct {
1545                 in  interface{}
1546                 out string
1547         }{
1548                 {in: "foo", out: `foo`},
1549                 {in: nil, out: "error"},
1550                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1551         } {
1552                 i = 0
1553                 cr.ArvMountPoint = ""
1554                 cr.Container.Mounts = map[string]arvados.Mount{
1555                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1556                 }
1557                 bindmounts, err := cr.SetupMounts()
1558                 if test.out == "error" {
1559                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1560                 } else {
1561                         c.Check(err, IsNil)
1562                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1563                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1564                         })
1565                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1566                         c.Check(err, IsNil)
1567                         c.Check(content, DeepEquals, []byte(test.out))
1568                 }
1569                 os.RemoveAll(cr.ArvMountPoint)
1570                 cr.CleanupDirs()
1571                 checkEmpty()
1572         }
1573
1574         // Read-only mount points are allowed underneath output_dir mount point
1575         {
1576                 i = 0
1577                 cr.ArvMountPoint = ""
1578                 cr.Container.Mounts = make(map[string]arvados.Mount)
1579                 cr.Container.Mounts = map[string]arvados.Mount{
1580                         "/tmp":     {Kind: "tmp"},
1581                         "/tmp/foo": {Kind: "collection"},
1582                 }
1583                 cr.Container.OutputPath = "/tmp"
1584
1585                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1586
1587                 bindmounts, err := cr.SetupMounts()
1588                 c.Check(err, IsNil)
1589                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1590                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1591                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1592                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1593                         "/tmp":     {realTemp + "/tmp2", false},
1594                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1595                 })
1596                 os.RemoveAll(cr.ArvMountPoint)
1597                 cr.CleanupDirs()
1598                 checkEmpty()
1599         }
1600
1601         // Writable mount points copied to output_dir mount point
1602         {
1603                 i = 0
1604                 cr.ArvMountPoint = ""
1605                 cr.Container.Mounts = make(map[string]arvados.Mount)
1606                 cr.Container.Mounts = map[string]arvados.Mount{
1607                         "/tmp": {Kind: "tmp"},
1608                         "/tmp/foo": {Kind: "collection",
1609                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1610                                 Writable:         true},
1611                         "/tmp/bar": {Kind: "collection",
1612                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1613                                 Path:             "baz",
1614                                 Writable:         true},
1615                 }
1616                 cr.Container.OutputPath = "/tmp"
1617
1618                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1619                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1620
1621                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1622                 rf.Write([]byte("bar"))
1623                 rf.Close()
1624
1625                 _, err := cr.SetupMounts()
1626                 c.Check(err, IsNil)
1627                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1628                 c.Check(err, IsNil)
1629                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1630                 c.Check(err, IsNil)
1631                 os.RemoveAll(cr.ArvMountPoint)
1632                 cr.CleanupDirs()
1633                 checkEmpty()
1634         }
1635
1636         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1637         {
1638                 i = 0
1639                 cr.ArvMountPoint = ""
1640                 cr.Container.Mounts = make(map[string]arvados.Mount)
1641                 cr.Container.Mounts = map[string]arvados.Mount{
1642                         "/tmp":     {Kind: "tmp"},
1643                         "/tmp/foo": {Kind: "tmp"},
1644                 }
1645                 cr.Container.OutputPath = "/tmp"
1646
1647                 _, err := cr.SetupMounts()
1648                 c.Check(err, NotNil)
1649                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1650                 os.RemoveAll(cr.ArvMountPoint)
1651                 cr.CleanupDirs()
1652                 checkEmpty()
1653         }
1654
1655         // Only mount point of kind 'collection' is allowed for stdin
1656         {
1657                 i = 0
1658                 cr.ArvMountPoint = ""
1659                 cr.Container.Mounts = make(map[string]arvados.Mount)
1660                 cr.Container.Mounts = map[string]arvados.Mount{
1661                         "stdin": {Kind: "tmp"},
1662                 }
1663
1664                 _, err := cr.SetupMounts()
1665                 c.Check(err, NotNil)
1666                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1667                 os.RemoveAll(cr.ArvMountPoint)
1668                 cr.CleanupDirs()
1669                 checkEmpty()
1670         }
1671
1672         // git_tree mounts
1673         {
1674                 i = 0
1675                 cr.ArvMountPoint = ""
1676                 (*GitMountSuite)(nil).useTestGitServer(c)
1677                 cr.token = arvadostest.ActiveToken
1678                 cr.Container.Mounts = make(map[string]arvados.Mount)
1679                 cr.Container.Mounts = map[string]arvados.Mount{
1680                         "/tip": {
1681                                 Kind:   "git_tree",
1682                                 UUID:   arvadostest.Repository2UUID,
1683                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1684                                 Path:   "/",
1685                         },
1686                         "/non-tip": {
1687                                 Kind:   "git_tree",
1688                                 UUID:   arvadostest.Repository2UUID,
1689                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1690                                 Path:   "/",
1691                         },
1692                 }
1693                 cr.Container.OutputPath = "/tmp"
1694
1695                 bindmounts, err := cr.SetupMounts()
1696                 c.Check(err, IsNil)
1697
1698                 for path, mount := range bindmounts {
1699                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1700                 }
1701
1702                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1703                 c.Check(err, IsNil)
1704                 c.Check(string(data), Equals, "\000\001\002\003")
1705                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1706                 c.Check(err, FitsTypeOf, &os.PathError{})
1707                 c.Check(os.IsNotExist(err), Equals, true)
1708
1709                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1710                 c.Check(err, IsNil)
1711                 c.Check(string(data), Equals, "\000\001\002\003")
1712                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1713                 c.Check(err, IsNil)
1714                 c.Check(string(data), Equals, "testfile\n")
1715
1716                 cr.CleanupDirs()
1717                 checkEmpty()
1718         }
1719 }
1720
1721 func (s *TestSuite) TestStdout(c *C) {
1722         helperRecord := `{
1723                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1724                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1725                 "cwd": "/bin",
1726                 "environment": {"FROBIZ": "bilbo"},
1727                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1728                 "output_path": "/tmp",
1729                 "priority": 1,
1730                 "runtime_constraints": {},
1731                 "state": "Locked"
1732         }`
1733
1734         s.fullRunHelper(c, helperRecord, nil, func() int {
1735                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1736                 return 0
1737         })
1738
1739         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1740         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1741         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1742 }
1743
1744 // Used by the TestStdoutWithWrongPath*()
1745 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func() int) (*ArvTestClient, *ContainerRunner, error) {
1746         err := json.Unmarshal([]byte(record), &s.api.Container)
1747         c.Assert(err, IsNil)
1748         s.executor.runFunc = fn
1749         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1750         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1751                 return s.api, &KeepTestClient{}, nil, nil
1752         }
1753         return s.api, s.runner, s.runner.Run()
1754 }
1755
1756 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1757         _, _, err := s.stdoutErrorRunHelper(c, `{
1758     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1759     "output_path": "/tmp",
1760     "state": "Locked"
1761 }`, func() int { return 0 })
1762         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1763 }
1764
1765 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1766         _, _, err := s.stdoutErrorRunHelper(c, `{
1767     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1768     "output_path": "/tmp",
1769     "state": "Locked"
1770 }`, func() int { return 0 })
1771         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1772 }
1773
1774 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1775         _, _, err := s.stdoutErrorRunHelper(c, `{
1776     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1777     "output_path": "/tmp",
1778     "state": "Locked"
1779 }`, func() int { return 0 })
1780         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1781 }
1782
1783 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1784         s.fullRunHelper(c, `{
1785     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1786     "container_image": "`+arvadostest.DockerImage112PDH+`",
1787     "cwd": "/bin",
1788     "environment": {},
1789     "mounts": {"/tmp": {"kind": "tmp"} },
1790     "output_path": "/tmp",
1791     "priority": 1,
1792     "runtime_constraints": {"API": true},
1793     "state": "Locked"
1794 }`, nil, func() int {
1795                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1796                 return 3
1797         })
1798         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1799         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1800         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1801 }
1802
1803 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1804         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1805         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1806         s.fullRunHelper(c, `{
1807     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1808     "container_image": "`+arvadostest.DockerImage112PDH+`",
1809     "cwd": "/bin",
1810     "environment": {},
1811     "mounts": {"/tmp": {"kind": "tmp"} },
1812     "output_path": "/tmp",
1813     "priority": 1,
1814     "runtime_constraints": {"API": true},
1815     "state": "Locked"
1816 }`, nil, func() int {
1817                 s.api.Container.Output = arvadostest.DockerImage112PDH
1818                 return 0
1819         })
1820
1821         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1822         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1823         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1824 }
1825
1826 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1827         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1828                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1829                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1830                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1831         }
1832         s.executor.runFunc = func() int {
1833                 time.Sleep(time.Second)
1834                 return 137
1835         }
1836         record := `{
1837     "command": ["sleep", "1"],
1838     "container_image": "` + arvadostest.DockerImage112PDH + `",
1839     "cwd": "/bin",
1840     "environment": {},
1841     "mounts": {"/tmp": {"kind": "tmp"} },
1842     "output_path": "/tmp",
1843     "priority": 1,
1844     "runtime_constraints": {"API": true},
1845     "state": "Locked"
1846 }`
1847         err := json.Unmarshal([]byte(record), &s.api.Container)
1848         c.Assert(err, IsNil)
1849         err = s.runner.Run()
1850         c.Assert(err, IsNil)
1851         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1852         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1853         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1854         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1855         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1856 }
1857
1858 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1859         helperRecord := `{
1860                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1861                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1862                 "cwd": "/bin",
1863                 "environment": {"FROBIZ": "bilbo"},
1864                 "mounts": {
1865         "/tmp": {"kind": "tmp"},
1866         "/tmp/foo": {"kind": "collection",
1867                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1868                      "exclude_from_output": true
1869         },
1870         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1871     },
1872                 "output_path": "/tmp",
1873                 "priority": 1,
1874                 "runtime_constraints": {},
1875                 "state": "Locked"
1876         }`
1877
1878         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1879
1880         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1881                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1882                 return 0
1883         })
1884
1885         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1886         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1887         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1888 }
1889
1890 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1891         helperRecord := `{
1892                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1893                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1894                 "cwd": "/bin",
1895                 "environment": {"FROBIZ": "bilbo"},
1896                 "mounts": {
1897         "/tmp": {"kind": "tmp"},
1898         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1899         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1900         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1901         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1902         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1903     },
1904                 "output_path": "/tmp",
1905                 "priority": 1,
1906                 "runtime_constraints": {},
1907                 "state": "Locked",
1908                 "uuid": "zzzzz-dz642-202301130848001"
1909         }`
1910
1911         extraMounts := []string{
1912                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1913                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1914                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1915         }
1916
1917         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1918                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1919                 return 0
1920         })
1921
1922         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1923                 "/tmp":                   {realtemp + "/tmp1", false},
1924                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1925                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1926                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1927                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1928         })
1929
1930         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1931         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1932         output_count := uint(0)
1933         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1934                 if v["collection"] == nil {
1935                         continue
1936                 }
1937                 collection := v["collection"].(arvadosclient.Dict)
1938                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
1939                         continue
1940                 }
1941                 c.Check(v["ensure_unique_name"], Equals, true)
1942                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1943 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1944 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1945 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1946 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1947 `)
1948                 output_count++
1949         }
1950         c.Check(output_count, Not(Equals), uint(0))
1951 }
1952
1953 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1954         helperRecord := `{
1955                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1956                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1957                 "cwd": "/bin",
1958                 "environment": {"FROBIZ": "bilbo"},
1959                 "mounts": {
1960         "/tmp": {"kind": "tmp"},
1961         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1962         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1963     },
1964                 "output_path": "/tmp",
1965                 "priority": 1,
1966                 "runtime_constraints": {},
1967                 "state": "Locked",
1968                 "uuid": "zzzzz-dz642-202301130848002"
1969         }`
1970
1971         extraMounts := []string{
1972                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1973         }
1974
1975         s.fullRunHelper(c, helperRecord, extraMounts, func() int {
1976                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1977                 return 0
1978         })
1979
1980         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1981         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1982         output_count := uint(0)
1983         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1984                 if v["collection"] == nil {
1985                         continue
1986                 }
1987                 collection := v["collection"].(arvadosclient.Dict)
1988                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
1989                         continue
1990                 }
1991                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1992 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1993 `)
1994                 output_count++
1995         }
1996         c.Check(output_count, Not(Equals), uint(0))
1997 }
1998
1999 func (s *TestSuite) TestOutputError(c *C) {
2000         helperRecord := `{
2001                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2002                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2003                 "cwd": "/bin",
2004                 "environment": {"FROBIZ": "bilbo"},
2005                 "mounts": {
2006                         "/tmp": {"kind": "tmp"}
2007                 },
2008                 "output_path": "/tmp",
2009                 "priority": 1,
2010                 "runtime_constraints": {},
2011                 "state": "Locked"
2012         }`
2013         s.fullRunHelper(c, helperRecord, nil, func() int {
2014                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
2015                 return 0
2016         })
2017
2018         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2019 }
2020
2021 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
2022         helperRecord := `{
2023                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2024                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2025                 "cwd": "/bin",
2026                 "environment": {"FROBIZ": "bilbo"},
2027                 "mounts": {
2028         "/tmp": {"kind": "tmp"},
2029         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
2030         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2031     },
2032                 "output_path": "/tmp",
2033                 "priority": 1,
2034                 "runtime_constraints": {},
2035                 "state": "Locked"
2036         }`
2037
2038         extraMounts := []string{
2039                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
2040         }
2041
2042         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, func() int {
2043                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2044                 return 0
2045         })
2046
2047         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2048         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2049         for _, v := range api.Content {
2050                 if v["collection"] != nil {
2051                         collection := v["collection"].(arvadosclient.Dict)
2052                         if strings.Index(collection["name"].(string), "output") == 0 {
2053                                 manifest := collection["manifest_text"].(string)
2054                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2055 `)
2056                         }
2057                 }
2058         }
2059 }
2060
2061 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
2062         helperRecord := `{
2063                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
2064                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2065                 "cwd": "/bin",
2066                 "environment": {"FROBIZ": "bilbo"},
2067                 "mounts": {
2068         "/tmp": {"kind": "tmp"},
2069         "stdin": {"kind": "json", "content": "foo"},
2070         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
2071     },
2072                 "output_path": "/tmp",
2073                 "priority": 1,
2074                 "runtime_constraints": {},
2075                 "state": "Locked"
2076         }`
2077
2078         api, _, _ := s.fullRunHelper(c, helperRecord, nil, func() int {
2079                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
2080                 return 0
2081         })
2082
2083         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
2084         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
2085         for _, v := range api.Content {
2086                 if v["collection"] != nil {
2087                         collection := v["collection"].(arvadosclient.Dict)
2088                         if strings.Index(collection["name"].(string), "output") == 0 {
2089                                 manifest := collection["manifest_text"].(string)
2090                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
2091 `)
2092                         }
2093                 }
2094         }
2095 }
2096
2097 func (s *TestSuite) TestStderrMount(c *C) {
2098         api, cr, _ := s.fullRunHelper(c, `{
2099     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
2100     "container_image": "`+arvadostest.DockerImage112PDH+`",
2101     "cwd": ".",
2102     "environment": {},
2103     "mounts": {"/tmp": {"kind": "tmp"},
2104                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
2105                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
2106     "output_path": "/tmp",
2107     "priority": 1,
2108     "runtime_constraints": {},
2109     "state": "Locked"
2110 }`, nil, func() int {
2111                 fmt.Fprintln(s.executor.created.Stdout, "hello")
2112                 fmt.Fprintln(s.executor.created.Stderr, "oops")
2113                 return 1
2114         })
2115
2116         final := api.CalledWith("container.state", "Complete")
2117         c.Assert(final, NotNil)
2118         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
2119         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
2120
2121         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
2122 }
2123
2124 func (s *TestSuite) TestNumberRoundTrip(c *C) {
2125         s.api.callraw = true
2126         err := s.runner.fetchContainerRecord()
2127         c.Assert(err, IsNil)
2128         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
2129         c.Logf("%#v", s.runner.Container)
2130         c.Check(err, IsNil)
2131         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
2132 }
2133
2134 func (s *TestSuite) TestFullBrokenDocker(c *C) {
2135         nextState := ""
2136         for _, setup := range []func(){
2137                 func() {
2138                         c.Log("// waitErr = ocl runtime error")
2139                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
2140                         nextState = "Cancelled"
2141                 },
2142                 func() {
2143                         c.Log("// loadErr = cannot connect")
2144                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
2145                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
2146                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
2147                         c.Assert(err, IsNil)
2148                         nextState = "Queued"
2149                 },
2150         } {
2151                 s.SetUpTest(c)
2152                 setup()
2153                 s.fullRunHelper(c, `{
2154     "command": ["echo", "hello world"],
2155     "container_image": "`+arvadostest.DockerImage112PDH+`",
2156     "cwd": ".",
2157     "environment": {},
2158     "mounts": {"/tmp": {"kind": "tmp"} },
2159     "output_path": "/tmp",
2160     "priority": 1,
2161     "runtime_constraints": {},
2162     "state": "Locked"
2163 }`, nil, func() int { return 0 })
2164                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
2165                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
2166                 if s.runner.brokenNodeHook != "" {
2167                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
2168                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
2169                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2170                 } else {
2171                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
2172                 }
2173         }
2174 }
2175
2176 func (s *TestSuite) TestBadCommand(c *C) {
2177         for _, startError := range []string{
2178                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
2179                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
2180                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2181         } {
2182                 s.SetUpTest(c)
2183                 s.executor.startErr = errors.New(startError)
2184                 s.fullRunHelper(c, `{
2185     "command": ["echo", "hello world"],
2186     "container_image": "`+arvadostest.DockerImage112PDH+`",
2187     "cwd": ".",
2188     "environment": {},
2189     "mounts": {"/tmp": {"kind": "tmp"} },
2190     "output_path": "/tmp",
2191     "priority": 1,
2192     "runtime_constraints": {},
2193     "state": "Locked"
2194 }`, nil, func() int { return 0 })
2195                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2196                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2197         }
2198 }
2199
2200 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2201         helperRecord := `{
2202                 "command": ["true"],
2203                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2204                 "cwd": "/bin",
2205                 "mounts": {
2206                     "/tmp": {"kind": "tmp"},
2207                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2208                 },
2209                 "secret_mounts": {
2210                 },
2211                 "output_path": "/tmp",
2212                 "priority": 1,
2213                 "runtime_constraints": {},
2214                 "state": "Locked"
2215         }`
2216
2217         s.fullRunHelper(c, helperRecord, nil, func() int {
2218                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2219                 c.Check(err, IsNil)
2220                 c.Check(string(content), Equals, "mypassword")
2221                 return 0
2222         })
2223
2224         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2225         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2226         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2227         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2228
2229         // under secret mounts, not captured in output
2230         helperRecord = `{
2231                 "command": ["true"],
2232                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2233                 "cwd": "/bin",
2234                 "mounts": {
2235                     "/tmp": {"kind": "tmp"}
2236                 },
2237                 "secret_mounts": {
2238                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2239                 },
2240                 "output_path": "/tmp",
2241                 "priority": 1,
2242                 "runtime_constraints": {},
2243                 "state": "Locked"
2244         }`
2245
2246         s.SetUpTest(c)
2247         s.fullRunHelper(c, helperRecord, nil, func() int {
2248                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2249                 c.Check(err, IsNil)
2250                 c.Check(string(content), Equals, "mypassword")
2251                 return 0
2252         })
2253
2254         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2255         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2256         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2257         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2258
2259         // under secret mounts, output dir is a collection, not captured in output
2260         helperRecord = `{
2261                 "command": ["true"],
2262                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2263                 "cwd": "/bin",
2264                 "mounts": {
2265                     "/tmp": {"kind": "collection", "writable": true}
2266                 },
2267                 "secret_mounts": {
2268                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2269                 },
2270                 "output_path": "/tmp",
2271                 "priority": 1,
2272                 "runtime_constraints": {},
2273                 "state": "Locked"
2274         }`
2275
2276         s.SetUpTest(c)
2277         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, func() int {
2278                 // secret.conf should be provisioned as a separate
2279                 // bind mount, i.e., it should not appear in the
2280                 // (fake) fuse filesystem as viewed from the host.
2281                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2282                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2283                         c.Logf("secret.conf: content %q, err %#v", content, err)
2284                 }
2285                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2286                 c.Check(err, IsNil)
2287                 return 0
2288         })
2289
2290         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2291         c.Check(err, IsNil)
2292         c.Check(string(content), Equals, "mypassword")
2293         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2294         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2295         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2296         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2297 }
2298
2299 func (s *TestSuite) TestCalculateCost(c *C) {
2300         defer func(s string) { lockdir = s }(lockdir)
2301         lockdir = c.MkDir()
2302         now := time.Now()
2303         cr := s.runner
2304         cr.costStartTime = now.Add(-time.Hour)
2305         var logbuf bytes.Buffer
2306         cr.CrunchLog.Immediate = log.New(&logbuf, "", 0)
2307
2308         // if there's no InstanceType env var, cost is calculated as 0
2309         os.Unsetenv("InstanceType")
2310         cost := cr.calculateCost(now)
2311         c.Check(cost, Equals, 0.0)
2312
2313         // with InstanceType env var and loadPrices() hasn't run (or
2314         // hasn't found any data), cost is calculated based on
2315         // InstanceType env var
2316         os.Setenv("InstanceType", `{"Price":1.2}`)
2317         defer os.Unsetenv("InstanceType")
2318         cost = cr.calculateCost(now)
2319         c.Check(cost, Equals, 1.2)
2320
2321         // first update tells us the spot price was $1/h until 30
2322         // minutes ago when it increased to $2/h
2323         j, err := json.Marshal([]cloud.InstancePrice{
2324                 {StartTime: now.Add(-4 * time.Hour), Price: 1.0},
2325                 {StartTime: now.Add(-time.Hour / 2), Price: 2.0},
2326         })
2327         c.Assert(err, IsNil)
2328         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2329         cr.loadPrices()
2330         cost = cr.calculateCost(now)
2331         c.Check(cost, Equals, 1.5)
2332
2333         // next update (via --list + SIGUSR2) tells us the spot price
2334         // increased to $3/h 15 minutes ago
2335         j, err = json.Marshal([]cloud.InstancePrice{
2336                 {StartTime: now.Add(-time.Hour / 3), Price: 2.0}, // dup of -time.Hour/2 price
2337                 {StartTime: now.Add(-time.Hour / 4), Price: 3.0},
2338         })
2339         c.Assert(err, IsNil)
2340         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2341         cr.loadPrices()
2342         cost = cr.calculateCost(now)
2343         c.Check(cost, Equals, 1.0/2+2.0/4+3.0/4)
2344
2345         cost = cr.calculateCost(now.Add(-time.Hour / 2))
2346         c.Check(cost, Equals, 0.5)
2347
2348         c.Logf("%s", logbuf.String())
2349         c.Check(logbuf.String(), Matches, `(?ms).*Instance price changed to 1\.00 at 20.* changed to 2\.00 .* changed to 3\.00 .*`)
2350         c.Check(logbuf.String(), Not(Matches), `(?ms).*changed to 2\.00 .* changed to 2\.00 .*`)
2351 }
2352
2353 type FakeProcess struct {
2354         cmdLine []string
2355 }
2356
2357 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2358         return fp.cmdLine, nil
2359 }