lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "log"
  16         "os"
  17         "os/exec"
  18         "regexp"
  19         "runtime/pprof"
  20         "strings"
  21         "sync"
  22         "syscall"
  23         "testing"
  24         "time"
  25
  26         "git.arvados.org/arvados.git/lib/cloud"
  27         "git.arvados.org/arvados.git/lib/cmd"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  30         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  31         "git.arvados.org/arvados.git/sdk/go/manifest"
  32         "golang.org/x/net/context"
  33
  34         . "gopkg.in/check.v1"
  35 )
  36
  37 // Gocheck boilerplate
  38 func TestCrunchExec(t *testing.T) {
  39         TestingT(t)
  40 }
  41
  42 var _ = Suite(&TestSuite{})
  43
  44 type TestSuite struct {
  45         client                   *arvados.Client
  46         api                      *ArvTestClient
  47         runner                   *ContainerRunner
  48         executor                 *stubExecutor
  49         keepmount                string
  50         keepmountTmp             []string
  51         testDispatcherKeepClient KeepTestClient
  52         testContainerKeepClient  KeepTestClient
  53 }
  54
  55 func (s *TestSuite) SetUpTest(c *C) {
  56         s.client = arvados.NewClientFromEnv()
  57         s.executor = &stubExecutor{}
  58         var err error
  59         s.api = &ArvTestClient{}
  60         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  61         c.Assert(err, IsNil)
  62         s.runner.executor = s.executor
  63         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  64                 return s.api, &s.testContainerKeepClient, s.client, nil
  65         }
  66         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  67                 s.runner.ArvMountPoint = s.keepmount
  68                 for i, opt := range cmd {
  69                         if opt == "--mount-tmp" {
  70                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  71                                 if err != nil {
  72                                         return nil, err
  73                                 }
  74                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  75                         }
  76                 }
  77                 return nil, nil
  78         }
  79         s.keepmount = c.MkDir()
  80         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  81         s.keepmountTmp = nil
  82         c.Assert(err, IsNil)
  83         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  84         c.Assert(err, IsNil)
  85         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  86         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  87         c.Assert(err, IsNil)
  88         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  89         c.Assert(err, IsNil)
  90         s.runner.ArvMountPoint = s.keepmount
  91 }
  92
  93 type ArvTestClient struct {
  94         Total   int64
  95         Calls   int
  96         Content []arvadosclient.Dict
  97         arvados.Container
  98         secretMounts []byte
  99         Logs         map[string]*bytes.Buffer
 100         sync.Mutex
 101         WasSetRunning bool
 102         callraw       bool
 103 }
 104
 105 type KeepTestClient struct {
 106         Called         bool
 107         Content        []byte
 108         StorageClasses []string
 109 }
 110
 111 type stubExecutor struct {
 112         imageLoaded bool
 113         loaded      string
 114         loadErr     error
 115         exitCode    int
 116         createErr   error
 117         created     containerSpec
 118         startErr    error
 119         waitSleep   time.Duration
 120         waitErr     error
 121         stopErr     error
 122         stopped     bool
 123         closed      bool
 124         runFunc     func()
 125         exit        chan int
 126 }
 127
 128 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 129         containerClient *arvados.Client) error {
 130         e.loaded = tarball
 131         return e.loadErr
 132 }
 133 func (e *stubExecutor) Runtime() string                 { return "stub" }
 134 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 135 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 136 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 137 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 138 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 139 func (e *stubExecutor) Close()                          { e.closed = true }
 140 func (e *stubExecutor) Wait(context.Context) (int, error) {
 141         return <-e.exit, e.waitErr
 142 }
 143 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 144         return nil, errors.New("unimplemented")
 145 }
 146 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 147
 148 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 149
 150 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 151 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 152 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 153
 154 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 155 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 156
 157 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 158 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 159 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 160 `
 161
 162 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 163
 164 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 165 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 166
 167 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 168 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 169
 170 func (client *ArvTestClient) Create(resourceType string,
 171         parameters arvadosclient.Dict,
 172         output interface{}) error {
 173
 174         client.Mutex.Lock()
 175         defer client.Mutex.Unlock()
 176
 177         client.Calls++
 178         client.Content = append(client.Content, parameters)
 179
 180         if resourceType == "logs" {
 181                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 182                 if client.Logs == nil {
 183                         client.Logs = make(map[string]*bytes.Buffer)
 184                 }
 185                 if client.Logs[et] == nil {
 186                         client.Logs[et] = &bytes.Buffer{}
 187                 }
 188                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 189         }
 190
 191         if resourceType == "collections" && output != nil {
 192                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 193                 md5sum := md5.Sum([]byte(mt))
 194                 outmap := output.(*arvados.Collection)
 195                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 196                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 197         }
 198
 199         return nil
 200 }
 201
 202 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 203         switch {
 204         case method == "GET" && resourceType == "containers" && action == "auth":
 205                 return json.Unmarshal([]byte(`{
 206                         "kind": "arvados#api_client_authorization",
 207                         "uuid": "`+fakeAuthUUID+`",
 208                         "api_token": "`+fakeAuthToken+`"
 209                         }`), output)
 210         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 211                 if client.secretMounts != nil {
 212                         return json.Unmarshal(client.secretMounts, output)
 213                 }
 214                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 215         default:
 216                 return fmt.Errorf("Not found")
 217         }
 218 }
 219
 220 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 221         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 222         var j []byte
 223         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 224                 j = []byte(`{
 225                         "kind": "arvados#nodeList",
 226                         "items": [{
 227                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 228                                 "hostname": "compute2",
 229                                 "properties": {"total_cpu_cores": 16}
 230                         }]}`)
 231         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 232                 if uuid == "" {
 233                         j, err = json.Marshal(map[string]interface{}{
 234                                 "items": []interface{}{client.Container},
 235                                 "kind":  "arvados#nodeList",
 236                         })
 237                 } else {
 238                         j, err = json.Marshal(client.Container)
 239                 }
 240         } else {
 241                 j = []byte(`{
 242                         "command": ["sleep", "1"],
 243                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 244                         "cwd": ".",
 245                         "environment": {},
 246                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 247                         "output_path": "/tmp",
 248                         "priority": 1,
 249                         "runtime_constraints": {}
 250                 }`)
 251         }
 252         return ioutil.NopCloser(bytes.NewReader(j)), err
 253 }
 254
 255 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 256         if resourceType == "collections" {
 257                 if uuid == hwPDH {
 258                         output.(*arvados.Collection).ManifestText = hwManifest
 259                 } else if uuid == otherPDH {
 260                         output.(*arvados.Collection).ManifestText = otherManifest
 261                 } else if uuid == normalizedWithSubdirsPDH {
 262                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 263                 } else if uuid == denormalizedWithSubdirsPDH {
 264                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 265                 }
 266         }
 267         if resourceType == "containers" {
 268                 (*output.(*arvados.Container)) = client.Container
 269         }
 270         return nil
 271 }
 272
 273 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 274         client.Mutex.Lock()
 275         defer client.Mutex.Unlock()
 276         client.Calls++
 277         client.Content = append(client.Content, parameters)
 278         if resourceType == "containers" {
 279                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 280                         client.WasSetRunning = true
 281                 }
 282         } else if resourceType == "collections" && output != nil {
 283                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 284                 output.(*arvados.Collection).UUID = uuid
 285                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 286         }
 287         return nil
 288 }
 289
 290 var discoveryMap = map[string]interface{}{
 291         "defaultTrashLifetime":               float64(1209600),
 292         "crunchLimitLogBytesPerJob":          float64(67108864),
 293         "crunchLogThrottleBytes":             float64(65536),
 294         "crunchLogThrottlePeriod":            float64(60),
 295         "crunchLogThrottleLines":             float64(1024),
 296         "crunchLogPartialLineThrottlePeriod": float64(5),
 297         "crunchLogBytesPerEvent":             float64(4096),
 298         "crunchLogSecondsBetweenEvents":      float64(1),
 299 }
 300
 301 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 302         return discoveryMap[key], nil
 303 }
 304
 305 // CalledWith returns the parameters from the first API call whose
 306 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 307 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 308 // no call matches, it returns nil.
 309 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 310 call:
 311         for _, content := range client.Content {
 312                 var v interface{} = content
 313                 for _, k := range strings.Split(jpath, ".") {
 314                         if dict, ok := v.(arvadosclient.Dict); !ok {
 315                                 continue call
 316                         } else {
 317                                 v = dict[k]
 318                         }
 319                 }
 320                 if v == expect {
 321                         return content
 322                 }
 323         }
 324         return nil
 325 }
 326
 327 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 328         return locator, nil
 329 }
 330
 331 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 332         client.Content = opts.Data
 333         return arvados.BlockWriteResponse{
 334                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 335         }, nil
 336 }
 337
 338 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 339         return 0, errors.New("not implemented")
 340 }
 341
 342 func (client *KeepTestClient) ClearBlockCache() {
 343 }
 344
 345 func (client *KeepTestClient) Close() {
 346         client.Content = nil
 347 }
 348
 349 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 350         client.StorageClasses = sc
 351 }
 352
 353 type FileWrapper struct {
 354         io.ReadCloser
 355         len int64
 356 }
 357
 358 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 359         return nil, errors.New("not implemented")
 360 }
 361
 362 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 363         return 0, errors.New("not implemented")
 364 }
 365
 366 func (fw FileWrapper) Size() int64 {
 367         return fw.len
 368 }
 369
 370 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 371         return nil, errors.New("not implemented")
 372 }
 373
 374 func (fw FileWrapper) Truncate(int64) error {
 375         return errors.New("not implemented")
 376 }
 377
 378 func (fw FileWrapper) Write([]byte) (int, error) {
 379         return 0, errors.New("not implemented")
 380 }
 381
 382 func (fw FileWrapper) Sync() error {
 383         return errors.New("not implemented")
 384 }
 385
 386 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 387         return nil, errors.New("not implemented")
 388 }
 389
 390 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 391         return errors.New("not implemented")
 392 }
 393
 394 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 395         if filename == hwImageID+".tar" {
 396                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 397                 client.Called = true
 398                 return FileWrapper{rdr, 1321984}, nil
 399         } else if filename == "/file1_in_main.txt" {
 400                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 401                 client.Called = true
 402                 return FileWrapper{rdr, 3}, nil
 403         }
 404         return nil, nil
 405 }
 406
 407 func (s *TestSuite) TestLoadImage(c *C) {
 408         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 409         s.runner.Container.Mounts = map[string]arvados.Mount{
 410                 "/out": {Kind: "tmp", Writable: true},
 411         }
 412         s.runner.Container.OutputPath = "/out"
 413
 414         _, err := s.runner.SetupMounts()
 415         c.Assert(err, IsNil)
 416
 417         imageID, err := s.runner.LoadImage()
 418         c.Check(err, IsNil)
 419         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 420         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 421
 422         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 423         s.executor.imageLoaded = false
 424         s.executor.loaded = ""
 425         s.executor.loadErr = errors.New("bork")
 426         imageID, err = s.runner.LoadImage()
 427         c.Check(err, ErrorMatches, ".*bork")
 428         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 429
 430         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 431         s.executor.imageLoaded = false
 432         s.executor.loaded = ""
 433         s.executor.loadErr = nil
 434         imageID, err = s.runner.LoadImage()
 435         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 436         c.Check(s.executor.loaded, Equals, "")
 437 }
 438
 439 type ArvErrorTestClient struct{}
 440
 441 func (ArvErrorTestClient) Create(resourceType string,
 442         parameters arvadosclient.Dict,
 443         output interface{}) error {
 444         return nil
 445 }
 446
 447 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 448         if method == "GET" && resourceType == "containers" && action == "auth" {
 449                 return nil
 450         }
 451         return errors.New("ArvError")
 452 }
 453
 454 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 455         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 456         return nil, errors.New("ArvError")
 457 }
 458
 459 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 460         return errors.New("ArvError")
 461 }
 462
 463 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 464         return nil
 465 }
 466
 467 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 468         return discoveryMap[key], nil
 469 }
 470
 471 type KeepErrorTestClient struct {
 472         KeepTestClient
 473 }
 474
 475 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 476         return nil, errors.New("KeepError")
 477 }
 478
 479 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 480         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 481 }
 482
 483 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 484         return "", errors.New("KeepError")
 485 }
 486
 487 type KeepReadErrorTestClient struct {
 488         KeepTestClient
 489 }
 490
 491 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 492         return 0, errors.New("KeepError")
 493 }
 494
 495 type ErrorReader struct {
 496         FileWrapper
 497 }
 498
 499 func (ErrorReader) Read(p []byte) (n int, err error) {
 500         return 0, errors.New("ErrorReader")
 501 }
 502
 503 func (ErrorReader) Seek(int64, int) (int64, error) {
 504         return 0, errors.New("ErrorReader")
 505 }
 506
 507 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 508         return ErrorReader{}, nil
 509 }
 510
 511 type ClosableBuffer struct {
 512         bytes.Buffer
 513 }
 514
 515 func (*ClosableBuffer) Close() error {
 516         return nil
 517 }
 518
 519 type TestLogs struct {
 520         Stdout ClosableBuffer
 521         Stderr ClosableBuffer
 522 }
 523
 524 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 525         if logstr == "stdout" {
 526                 return &tl.Stdout, nil
 527         }
 528         if logstr == "stderr" {
 529                 return &tl.Stderr, nil
 530         }
 531         return nil, errors.New("???")
 532 }
 533
 534 func dockerLog(fd byte, msg string) []byte {
 535         by := []byte(msg)
 536         header := make([]byte, 8+len(by))
 537         header[0] = fd
 538         header[7] = byte(len(by))
 539         copy(header[8:], by)
 540         return header
 541 }
 542
 543 func (s *TestSuite) TestRunContainer(c *C) {
 544         s.executor.runFunc = func() {
 545                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 546                 s.executor.exit <- 0
 547         }
 548
 549         var logs TestLogs
 550         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 551         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 552         s.runner.Container.Command = []string{"./hw"}
 553         s.runner.Container.OutputStorageClasses = []string{"default"}
 554
 555         imageID, err := s.runner.LoadImage()
 556         c.Assert(err, IsNil)
 557
 558         err = s.runner.CreateContainer(imageID, nil)
 559         c.Assert(err, IsNil)
 560
 561         err = s.runner.StartContainer()
 562         c.Assert(err, IsNil)
 563
 564         err = s.runner.WaitFinish()
 565         c.Assert(err, IsNil)
 566
 567         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 568         c.Check(logs.Stderr.String(), Equals, "")
 569 }
 570
 571 func (s *TestSuite) TestCommitLogs(c *C) {
 572         api := &ArvTestClient{}
 573         kc := &KeepTestClient{}
 574         defer kc.Close()
 575         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 576         c.Assert(err, IsNil)
 577         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 578
 579         cr.CrunchLog.Print("Hello world!")
 580         cr.CrunchLog.Print("Goodbye")
 581         cr.finalState = "Complete"
 582
 583         err = cr.CommitLogs()
 584         c.Check(err, IsNil)
 585
 586         c.Check(api.Calls, Equals, 2)
 587         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 588         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 589         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 590         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 591 }
 592
 593 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 594         api := &ArvTestClient{}
 595         kc := &KeepTestClient{}
 596         defer kc.Close()
 597         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 598         c.Assert(err, IsNil)
 599
 600         err = cr.UpdateContainerRunning("")
 601         c.Check(err, IsNil)
 602
 603         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 604 }
 605
 606 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 607         api := &ArvTestClient{}
 608         kc := &KeepTestClient{}
 609         defer kc.Close()
 610         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 611         c.Assert(err, IsNil)
 612
 613         cr.LogsPDH = new(string)
 614         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 615
 616         cr.ExitCode = new(int)
 617         *cr.ExitCode = 42
 618         cr.finalState = "Complete"
 619
 620         err = cr.UpdateContainerFinal()
 621         c.Check(err, IsNil)
 622
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 624         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 625         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 626 }
 627
 628 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 629         api := &ArvTestClient{}
 630         kc := &KeepTestClient{}
 631         defer kc.Close()
 632         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 633         c.Assert(err, IsNil)
 634         cr.cCancelled = true
 635         cr.finalState = "Cancelled"
 636
 637         err = cr.UpdateContainerFinal()
 638         c.Check(err, IsNil)
 639
 640         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 641         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 642         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 643 }
 644
 645 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 646 // dress rehearsal of the Run() function, starting from a JSON container record.
 647 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 648         err := json.Unmarshal([]byte(record), &s.api.Container)
 649         c.Assert(err, IsNil)
 650         initialState := s.api.Container.State
 651
 652         var sm struct {
 653                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 654         }
 655         err = json.Unmarshal([]byte(record), &sm)
 656         c.Check(err, IsNil)
 657         secretMounts, err := json.Marshal(sm)
 658         c.Assert(err, IsNil)
 659         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 660
 661         s.executor.runFunc = func() {
 662                 fn()
 663                 s.executor.exit <- exitCode
 664         }
 665
 666         s.runner.statInterval = 100 * time.Millisecond
 667         s.runner.containerWatchdogInterval = time.Second
 668
 669         realTemp := c.MkDir()
 670         tempcount := 0
 671         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 672                 tempcount++
 673                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 674                 err := os.Mkdir(d, os.ModePerm)
 675                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 676                         // Test case must have pre-populated the tempdir
 677                         err = nil
 678                 }
 679                 return d, err
 680         }
 681         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 682                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 683         }
 684
 685         if extraMounts != nil && len(extraMounts) > 0 {
 686                 err := s.runner.SetupArvMountPoint("keep")
 687                 c.Check(err, IsNil)
 688
 689                 for _, m := range extraMounts {
 690                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 691                 }
 692         }
 693
 694         err = s.runner.Run()
 695         if s.api.CalledWith("container.state", "Complete") != nil {
 696                 c.Check(err, IsNil)
 697         }
 698         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 699                 c.Check(s.api.WasSetRunning, Equals, true)
 700                 var lastupdate arvadosclient.Dict
 701                 for _, content := range s.api.Content {
 702                         if content["container"] != nil {
 703                                 lastupdate = content["container"].(arvadosclient.Dict)
 704                         }
 705                 }
 706                 if lastupdate["log"] == nil {
 707                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 708                 }
 709         }
 710
 711         if err != nil {
 712                 for k, v := range s.api.Logs {
 713                         c.Log(k)
 714                         c.Log(v.String())
 715                 }
 716         }
 717
 718         return s.api, s.runner, realTemp
 719 }
 720
 721 func (s *TestSuite) TestFullRunHello(c *C) {
 722         s.runner.enableMemoryLimit = true
 723         s.runner.networkMode = "default"
 724         s.fullRunHelper(c, `{
 725     "command": ["echo", "hello world"],
 726     "container_image": "`+arvadostest.DockerImage112PDH+`",
 727     "cwd": ".",
 728     "environment": {"foo":"bar","baz":"waz"},
 729     "mounts": {"/tmp": {"kind": "tmp"} },
 730     "output_path": "/tmp",
 731     "priority": 1,
 732     "runtime_constraints": {"vcpus":1,"ram":1000000},
 733     "state": "Locked",
 734     "output_storage_classes": ["default"]
 735 }`, nil, 0, func() {
 736                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 737                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 738                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 739                 c.Check(s.executor.created.VCPUs, Equals, 1)
 740                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 741                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 742                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 743                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 744                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 745         })
 746
 747         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 748         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 749         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 750         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 751         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 752 }
 753
 754 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 755         var ran bool
 756         s.fullRunHelper(c, `{
 757     "command": ["sleep", "3"],
 758     "container_image": "`+arvadostest.DockerImage112PDH+`",
 759     "cwd": ".",
 760     "environment": {},
 761     "mounts": {"/tmp": {"kind": "tmp"} },
 762     "output_path": "/tmp",
 763     "priority": 1,
 764     "runtime_constraints": {},
 765     "scheduling_parameters":{"max_run_time": 1},
 766     "state": "Running"
 767 }`, nil, 2, func() {
 768                 ran = true
 769         })
 770         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 771         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 772         c.Check(ran, Equals, false)
 773 }
 774
 775 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 776         s.fullRunHelper(c, `{
 777     "command": ["sleep", "3"],
 778     "container_image": "`+arvadostest.DockerImage112PDH+`",
 779     "cwd": ".",
 780     "environment": {},
 781     "mounts": {"/tmp": {"kind": "tmp"} },
 782     "output_path": "/tmp",
 783     "priority": 1,
 784     "runtime_constraints": {},
 785     "scheduling_parameters":{"max_run_time": 1},
 786     "state": "Locked"
 787 }`, nil, 0, func() {
 788                 time.Sleep(3 * time.Second)
 789         })
 790
 791         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 792         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 793 }
 794
 795 func (s *TestSuite) TestContainerWaitFails(c *C) {
 796         s.fullRunHelper(c, `{
 797     "command": ["sleep", "3"],
 798     "container_image": "`+arvadostest.DockerImage112PDH+`",
 799     "cwd": ".",
 800     "mounts": {"/tmp": {"kind": "tmp"} },
 801     "output_path": "/tmp",
 802     "priority": 1,
 803     "state": "Locked"
 804 }`, nil, 0, func() {
 805                 s.executor.waitErr = errors.New("Container is not running")
 806         })
 807
 808         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 809         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 810 }
 811
 812 func (s *TestSuite) TestCrunchstat(c *C) {
 813         s.fullRunHelper(c, `{
 814                 "command": ["sleep", "1"],
 815                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 816                 "cwd": ".",
 817                 "environment": {},
 818                 "mounts": {"/tmp": {"kind": "tmp"} },
 819                 "output_path": "/tmp",
 820                 "priority": 1,
 821                 "runtime_constraints": {},
 822                 "state": "Locked"
 823         }`, nil, 0, func() {
 824                 time.Sleep(time.Second)
 825         })
 826
 827         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 828         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 829
 830         // We didn't actually start a container, so crunchstat didn't
 831         // find accounting files and therefore didn't log any stats.
 832         // It should have logged a "can't find accounting files"
 833         // message after one poll interval, though, so we can confirm
 834         // it's alive:
 835         c.Assert(s.api.Logs["crunchstat"], NotNil)
 836         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 837
 838         // The "files never appeared" log assures us that we called
 839         // (*crunchstat.Reporter)Stop(), and that we set it up with
 840         // the correct container ID "abcde":
 841         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 842 }
 843
 844 func (s *TestSuite) TestNodeInfoLog(c *C) {
 845         os.Setenv("SLURMD_NODENAME", "compute2")
 846         s.fullRunHelper(c, `{
 847                 "command": ["sleep", "1"],
 848                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 849                 "cwd": ".",
 850                 "environment": {},
 851                 "mounts": {"/tmp": {"kind": "tmp"} },
 852                 "output_path": "/tmp",
 853                 "priority": 1,
 854                 "runtime_constraints": {},
 855                 "state": "Locked"
 856         }`, nil, 0,
 857                 func() {
 858                         time.Sleep(time.Second)
 859                 })
 860
 861         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 862         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 863
 864         c.Assert(s.api.Logs["node"], NotNil)
 865         json := s.api.Logs["node"].String()
 866         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 867         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 868         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 869
 870         c.Assert(s.api.Logs["node-info"], NotNil)
 871         json = s.api.Logs["node-info"].String()
 872         c.Check(json, Matches, `(?ms).*Host Information.*`)
 873         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 874         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 875         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 876         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 877 }
 878
 879 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 880         s.fullRunHelper(c, `{
 881                 "command": ["sleep", "1"],
 882                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 883                 "cwd": ".",
 884                 "environment": {},
 885                 "mounts": {"/tmp": {"kind": "tmp"} },
 886                 "output_path": "/tmp",
 887                 "priority": 1,
 888                 "runtime_constraints": {},
 889                 "state": "Locked"
 890         }`, nil, 0,
 891                 func() {
 892                 })
 893
 894         c.Assert(s.api.Logs["crunch-run"], NotNil)
 895         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 896         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 897         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 898         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 899 }
 900
 901 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
 902         var collection_create, container_update arvadosclient.Dict
 903         s.fullRunHelper(c, `{
 904                 "command": ["true"],
 905                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 906                 "cwd": ".",
 907                 "environment": {},
 908                 "mounts": {"/tmp": {"kind": "tmp"} },
 909                 "output_path": "/tmp",
 910                 "priority": 1,
 911                 "runtime_constraints": {},
 912                 "state": "Locked",
 913                 "uuid": "zzzzz-dz642-202301121543210"
 914         }`, nil, 0,
 915                 func() {
 916                         collection_create = s.api.CalledWith("ensure_unique_name", true)
 917                         container_update = s.api.CalledWith("container.state", "Running")
 918                 })
 919
 920         c.Assert(collection_create, NotNil)
 921         log_collection := collection_create["collection"].(arvadosclient.Dict)
 922         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
 923         manifest_text := log_collection["manifest_text"].(string)
 924         // We check that the file size is at least two digits as an easy way to
 925         // check the file isn't empty.
 926         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node-info\.txt( .+)?\n`)
 927         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node\.json( .+)?\n`)
 928
 929         c.Assert(container_update, NotNil)
 930         // As of Arvados 2.5.0, the container update must specify its log in PDH
 931         // format for the API server to propagate it to container requests, which
 932         // is what we care about for this test.
 933         expect_pdh := fmt.Sprintf("%x+%d", md5.Sum([]byte(manifest_text)), len(manifest_text))
 934         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Equals, expect_pdh)
 935 }
 936
 937 func (s *TestSuite) TestContainerRecordLog(c *C) {
 938         s.fullRunHelper(c, `{
 939                 "command": ["sleep", "1"],
 940                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 941                 "cwd": ".",
 942                 "environment": {},
 943                 "mounts": {"/tmp": {"kind": "tmp"} },
 944                 "output_path": "/tmp",
 945                 "priority": 1,
 946                 "runtime_constraints": {},
 947                 "state": "Locked"
 948         }`, nil, 0,
 949                 func() {
 950                         time.Sleep(time.Second)
 951                 })
 952
 953         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 954         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 955
 956         c.Assert(s.api.Logs["container"], NotNil)
 957         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 958 }
 959
 960 func (s *TestSuite) TestFullRunStderr(c *C) {
 961         s.fullRunHelper(c, `{
 962     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 963     "container_image": "`+arvadostest.DockerImage112PDH+`",
 964     "cwd": ".",
 965     "environment": {},
 966     "mounts": {"/tmp": {"kind": "tmp"} },
 967     "output_path": "/tmp",
 968     "priority": 1,
 969     "runtime_constraints": {},
 970     "state": "Locked"
 971 }`, nil, 1, func() {
 972                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 973                 fmt.Fprintln(s.executor.created.Stderr, "world")
 974         })
 975
 976         final := s.api.CalledWith("container.state", "Complete")
 977         c.Assert(final, NotNil)
 978         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 979         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 980
 981         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 982         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 983 }
 984
 985 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 986         s.fullRunHelper(c, `{
 987     "command": ["pwd"],
 988     "container_image": "`+arvadostest.DockerImage112PDH+`",
 989     "cwd": ".",
 990     "environment": {},
 991     "mounts": {"/tmp": {"kind": "tmp"} },
 992     "output_path": "/tmp",
 993     "priority": 1,
 994     "runtime_constraints": {},
 995     "state": "Locked"
 996 }`, nil, 0, func() {
 997                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 998         })
 999
1000         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1001         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1002         c.Log(s.api.Logs["stdout"])
1003         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
1004 }
1005
1006 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1007         s.fullRunHelper(c, `{
1008     "command": ["pwd"],
1009     "container_image": "`+arvadostest.DockerImage112PDH+`",
1010     "cwd": "/bin",
1011     "environment": {},
1012     "mounts": {"/tmp": {"kind": "tmp"} },
1013     "output_path": "/tmp",
1014     "priority": 1,
1015     "runtime_constraints": {},
1016     "state": "Locked"
1017 }`, nil, 0, func() {
1018                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1019         })
1020
1021         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1022         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1023         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1024 }
1025
1026 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1027         s.fullRunHelper(c, `{
1028     "command": ["pwd"],
1029     "container_image": "`+arvadostest.DockerImage112PDH+`",
1030     "cwd": "/bin",
1031     "environment": {},
1032     "mounts": {"/tmp": {"kind": "tmp"} },
1033     "output_path": "/tmp",
1034     "priority": 1,
1035     "runtime_constraints": {},
1036     "state": "Locked",
1037     "output_storage_classes": ["foo", "bar"]
1038 }`, nil, 0, func() {
1039                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1040         })
1041
1042         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1043         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1044         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1045         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1046         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1047 }
1048
1049 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1050         s.fullRunHelper(c, `{
1051     "command": ["pwd"],
1052     "container_image": "`+arvadostest.DockerImage112PDH+`",
1053     "cwd": "/bin",
1054     "environment": {},
1055     "mounts": {"/tmp": {"kind": "tmp"} },
1056     "output_path": "/tmp",
1057     "priority": 1,
1058     "runtime_constraints": {"cuda": {"device_count": 2}},
1059     "state": "Locked",
1060     "output_storage_classes": ["foo", "bar"]
1061 }`, nil, 0, func() {
1062                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1063         })
1064         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1065 }
1066
1067 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1068         s.fullRunHelper(c, `{
1069     "command": ["pwd"],
1070     "container_image": "`+arvadostest.DockerImage112PDH+`",
1071     "cwd": "/bin",
1072     "environment": {},
1073     "mounts": {"/tmp": {"kind": "tmp"} },
1074     "output_path": "/tmp",
1075     "priority": 1,
1076     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1077     "state": "Locked",
1078     "output_storage_classes": ["foo", "bar"]
1079 }`, nil, 0, func() {
1080                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1081         })
1082         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1083 }
1084
1085 func (s *TestSuite) TestStopOnSignal(c *C) {
1086         s.executor.runFunc = func() {
1087                 s.executor.created.Stdout.Write([]byte("foo\n"))
1088                 s.runner.SigChan <- syscall.SIGINT
1089         }
1090         s.testStopContainer(c)
1091 }
1092
1093 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1094         s.executor.runFunc = func() {
1095                 s.executor.created.Stdout.Write([]byte("foo\n"))
1096                 s.runner.ArvMountExit <- nil
1097                 close(s.runner.ArvMountExit)
1098         }
1099         s.runner.ArvMountExit = make(chan error)
1100         s.testStopContainer(c)
1101 }
1102
1103 func (s *TestSuite) testStopContainer(c *C) {
1104         record := `{
1105     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1106     "container_image": "` + arvadostest.DockerImage112PDH + `",
1107     "cwd": ".",
1108     "environment": {},
1109     "mounts": {"/tmp": {"kind": "tmp"} },
1110     "output_path": "/tmp",
1111     "priority": 1,
1112     "runtime_constraints": {},
1113     "state": "Locked"
1114 }`
1115
1116         err := json.Unmarshal([]byte(record), &s.api.Container)
1117         c.Assert(err, IsNil)
1118
1119         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1120         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1121                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1122         }
1123
1124         done := make(chan error)
1125         go func() {
1126                 done <- s.runner.Run()
1127         }()
1128         select {
1129         case <-time.After(20 * time.Second):
1130                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1131                 c.Fatal("timed out")
1132         case err = <-done:
1133                 c.Check(err, IsNil)
1134         }
1135         for k, v := range s.api.Logs {
1136                 c.Log(k)
1137                 c.Log(v.String(), "\n")
1138         }
1139
1140         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1141         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1142         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1143 }
1144
1145 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1146         s.fullRunHelper(c, `{
1147     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1148     "container_image": "`+arvadostest.DockerImage112PDH+`",
1149     "cwd": "/bin",
1150     "environment": {"FROBIZ": "bilbo"},
1151     "mounts": {"/tmp": {"kind": "tmp"} },
1152     "output_path": "/tmp",
1153     "priority": 1,
1154     "runtime_constraints": {},
1155     "state": "Locked"
1156 }`, nil, 0, func() {
1157                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1158         })
1159
1160         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1161         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1162         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1163 }
1164
1165 type ArvMountCmdLine struct {
1166         Cmd   []string
1167         token string
1168 }
1169
1170 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1171         am.Cmd = c
1172         am.token = token
1173         return nil, nil
1174 }
1175
1176 func stubCert(temp string) string {
1177         path := temp + "/ca-certificates.crt"
1178         crt, _ := os.Create(path)
1179         crt.Close()
1180         arvadosclient.CertFiles = []string{path}
1181         return path
1182 }
1183
1184 func (s *TestSuite) TestSetupMounts(c *C) {
1185         cr := s.runner
1186         am := &ArvMountCmdLine{}
1187         cr.RunArvMount = am.ArvMountTest
1188         cr.ContainerArvClient = &ArvTestClient{}
1189         cr.ContainerKeepClient = &KeepTestClient{}
1190         cr.Container.OutputStorageClasses = []string{"default"}
1191
1192         realTemp := c.MkDir()
1193         certTemp := c.MkDir()
1194         stubCertPath := stubCert(certTemp)
1195         cr.parentTemp = realTemp
1196
1197         i := 0
1198         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1199                 i++
1200                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1201                 err := os.Mkdir(d, os.ModePerm)
1202                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1203                         // Test case must have pre-populated the tempdir
1204                         err = nil
1205                 }
1206                 return d, err
1207         }
1208
1209         checkEmpty := func() {
1210                 // Should be deleted.
1211                 _, err := os.Stat(realTemp)
1212                 c.Assert(os.IsNotExist(err), Equals, true)
1213
1214                 // Now recreate it for the next test.
1215                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1216         }
1217
1218         {
1219                 i = 0
1220                 cr.ArvMountPoint = ""
1221                 cr.Container.Mounts = make(map[string]arvados.Mount)
1222                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1223                 cr.Container.OutputPath = "/tmp"
1224                 cr.statInterval = 5 * time.Second
1225                 bindmounts, err := cr.SetupMounts()
1226                 c.Check(err, IsNil)
1227                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1228                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1229                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1230                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1231                 os.RemoveAll(cr.ArvMountPoint)
1232                 cr.CleanupDirs()
1233                 checkEmpty()
1234         }
1235
1236         {
1237                 i = 0
1238                 cr.ArvMountPoint = ""
1239                 cr.Container.Mounts = make(map[string]arvados.Mount)
1240                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1241                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1242                 cr.Container.OutputPath = "/out"
1243                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1244
1245                 bindmounts, err := cr.SetupMounts()
1246                 c.Check(err, IsNil)
1247                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1248                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1249                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1250                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1251                 os.RemoveAll(cr.ArvMountPoint)
1252                 cr.CleanupDirs()
1253                 checkEmpty()
1254         }
1255
1256         {
1257                 i = 0
1258                 cr.ArvMountPoint = ""
1259                 cr.Container.Mounts = make(map[string]arvados.Mount)
1260                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1261                 cr.Container.OutputPath = "/tmp"
1262                 cr.Container.RuntimeConstraints.API = true
1263                 cr.Container.OutputStorageClasses = []string{"default"}
1264
1265                 bindmounts, err := cr.SetupMounts()
1266                 c.Check(err, IsNil)
1267                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1268                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1269                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1270                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1271                 os.RemoveAll(cr.ArvMountPoint)
1272                 cr.CleanupDirs()
1273                 checkEmpty()
1274
1275                 cr.Container.RuntimeConstraints.API = false
1276         }
1277
1278         {
1279                 i = 0
1280                 cr.ArvMountPoint = ""
1281                 cr.Container.Mounts = map[string]arvados.Mount{
1282                         "/keeptmp": {Kind: "collection", Writable: true},
1283                 }
1284                 cr.Container.OutputPath = "/keeptmp"
1285
1286                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1287
1288                 bindmounts, err := cr.SetupMounts()
1289                 c.Check(err, IsNil)
1290                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1291                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1292                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1293                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1294                 os.RemoveAll(cr.ArvMountPoint)
1295                 cr.CleanupDirs()
1296                 checkEmpty()
1297         }
1298
1299         {
1300                 i = 0
1301                 cr.ArvMountPoint = ""
1302                 cr.Container.Mounts = map[string]arvados.Mount{
1303                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1304                         "/keepout": {Kind: "collection", Writable: true},
1305                 }
1306                 cr.Container.OutputPath = "/keepout"
1307
1308                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1309                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1310
1311                 bindmounts, err := cr.SetupMounts()
1312                 c.Check(err, IsNil)
1313                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1314                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1315                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1316                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1317                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1318                         "/keepout": {realTemp + "/keep1/tmp0", false},
1319                 })
1320                 os.RemoveAll(cr.ArvMountPoint)
1321                 cr.CleanupDirs()
1322                 checkEmpty()
1323         }
1324
1325         {
1326                 i = 0
1327                 cr.ArvMountPoint = ""
1328                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1329                 cr.Container.Mounts = map[string]arvados.Mount{
1330                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1331                         "/keepout": {Kind: "collection", Writable: true},
1332                 }
1333                 cr.Container.OutputPath = "/keepout"
1334
1335                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1336                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1337
1338                 bindmounts, err := cr.SetupMounts()
1339                 c.Check(err, IsNil)
1340                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1341                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1342                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1343                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1344                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1345                         "/keepout": {realTemp + "/keep1/tmp0", false},
1346                 })
1347                 os.RemoveAll(cr.ArvMountPoint)
1348                 cr.CleanupDirs()
1349                 checkEmpty()
1350         }
1351
1352         for _, test := range []struct {
1353                 in  interface{}
1354                 out string
1355         }{
1356                 {in: "foo", out: `"foo"`},
1357                 {in: nil, out: `null`},
1358                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1359         } {
1360                 i = 0
1361                 cr.ArvMountPoint = ""
1362                 cr.Container.Mounts = map[string]arvados.Mount{
1363                         "/mnt/test.json": {Kind: "json", Content: test.in},
1364                 }
1365                 bindmounts, err := cr.SetupMounts()
1366                 c.Check(err, IsNil)
1367                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1368                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1369                 })
1370                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1371                 c.Check(err, IsNil)
1372                 c.Check(content, DeepEquals, []byte(test.out))
1373                 os.RemoveAll(cr.ArvMountPoint)
1374                 cr.CleanupDirs()
1375                 checkEmpty()
1376         }
1377
1378         for _, test := range []struct {
1379                 in  interface{}
1380                 out string
1381         }{
1382                 {in: "foo", out: `foo`},
1383                 {in: nil, out: "error"},
1384                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1385         } {
1386                 i = 0
1387                 cr.ArvMountPoint = ""
1388                 cr.Container.Mounts = map[string]arvados.Mount{
1389                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1390                 }
1391                 bindmounts, err := cr.SetupMounts()
1392                 if test.out == "error" {
1393                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1394                 } else {
1395                         c.Check(err, IsNil)
1396                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1397                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1398                         })
1399                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1400                         c.Check(err, IsNil)
1401                         c.Check(content, DeepEquals, []byte(test.out))
1402                 }
1403                 os.RemoveAll(cr.ArvMountPoint)
1404                 cr.CleanupDirs()
1405                 checkEmpty()
1406         }
1407
1408         // Read-only mount points are allowed underneath output_dir mount point
1409         {
1410                 i = 0
1411                 cr.ArvMountPoint = ""
1412                 cr.Container.Mounts = make(map[string]arvados.Mount)
1413                 cr.Container.Mounts = map[string]arvados.Mount{
1414                         "/tmp":     {Kind: "tmp"},
1415                         "/tmp/foo": {Kind: "collection"},
1416                 }
1417                 cr.Container.OutputPath = "/tmp"
1418
1419                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1420
1421                 bindmounts, err := cr.SetupMounts()
1422                 c.Check(err, IsNil)
1423                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1424                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1425                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1426                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1427                         "/tmp":     {realTemp + "/tmp2", false},
1428                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1429                 })
1430                 os.RemoveAll(cr.ArvMountPoint)
1431                 cr.CleanupDirs()
1432                 checkEmpty()
1433         }
1434
1435         // Writable mount points copied to output_dir mount point
1436         {
1437                 i = 0
1438                 cr.ArvMountPoint = ""
1439                 cr.Container.Mounts = make(map[string]arvados.Mount)
1440                 cr.Container.Mounts = map[string]arvados.Mount{
1441                         "/tmp": {Kind: "tmp"},
1442                         "/tmp/foo": {Kind: "collection",
1443                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1444                                 Writable:         true},
1445                         "/tmp/bar": {Kind: "collection",
1446                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1447                                 Path:             "baz",
1448                                 Writable:         true},
1449                 }
1450                 cr.Container.OutputPath = "/tmp"
1451
1452                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1453                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1454
1455                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1456                 rf.Write([]byte("bar"))
1457                 rf.Close()
1458
1459                 _, err := cr.SetupMounts()
1460                 c.Check(err, IsNil)
1461                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1462                 c.Check(err, IsNil)
1463                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1464                 c.Check(err, IsNil)
1465                 os.RemoveAll(cr.ArvMountPoint)
1466                 cr.CleanupDirs()
1467                 checkEmpty()
1468         }
1469
1470         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1471         {
1472                 i = 0
1473                 cr.ArvMountPoint = ""
1474                 cr.Container.Mounts = make(map[string]arvados.Mount)
1475                 cr.Container.Mounts = map[string]arvados.Mount{
1476                         "/tmp":     {Kind: "tmp"},
1477                         "/tmp/foo": {Kind: "tmp"},
1478                 }
1479                 cr.Container.OutputPath = "/tmp"
1480
1481                 _, err := cr.SetupMounts()
1482                 c.Check(err, NotNil)
1483                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1484                 os.RemoveAll(cr.ArvMountPoint)
1485                 cr.CleanupDirs()
1486                 checkEmpty()
1487         }
1488
1489         // Only mount point of kind 'collection' is allowed for stdin
1490         {
1491                 i = 0
1492                 cr.ArvMountPoint = ""
1493                 cr.Container.Mounts = make(map[string]arvados.Mount)
1494                 cr.Container.Mounts = map[string]arvados.Mount{
1495                         "stdin": {Kind: "tmp"},
1496                 }
1497
1498                 _, err := cr.SetupMounts()
1499                 c.Check(err, NotNil)
1500                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1501                 os.RemoveAll(cr.ArvMountPoint)
1502                 cr.CleanupDirs()
1503                 checkEmpty()
1504         }
1505
1506         // git_tree mounts
1507         {
1508                 i = 0
1509                 cr.ArvMountPoint = ""
1510                 (*GitMountSuite)(nil).useTestGitServer(c)
1511                 cr.token = arvadostest.ActiveToken
1512                 cr.Container.Mounts = make(map[string]arvados.Mount)
1513                 cr.Container.Mounts = map[string]arvados.Mount{
1514                         "/tip": {
1515                                 Kind:   "git_tree",
1516                                 UUID:   arvadostest.Repository2UUID,
1517                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1518                                 Path:   "/",
1519                         },
1520                         "/non-tip": {
1521                                 Kind:   "git_tree",
1522                                 UUID:   arvadostest.Repository2UUID,
1523                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1524                                 Path:   "/",
1525                         },
1526                 }
1527                 cr.Container.OutputPath = "/tmp"
1528
1529                 bindmounts, err := cr.SetupMounts()
1530                 c.Check(err, IsNil)
1531
1532                 for path, mount := range bindmounts {
1533                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1534                 }
1535
1536                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1537                 c.Check(err, IsNil)
1538                 c.Check(string(data), Equals, "\000\001\002\003")
1539                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1540                 c.Check(err, FitsTypeOf, &os.PathError{})
1541                 c.Check(os.IsNotExist(err), Equals, true)
1542
1543                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1544                 c.Check(err, IsNil)
1545                 c.Check(string(data), Equals, "\000\001\002\003")
1546                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1547                 c.Check(err, IsNil)
1548                 c.Check(string(data), Equals, "testfile\n")
1549
1550                 cr.CleanupDirs()
1551                 checkEmpty()
1552         }
1553 }
1554
1555 func (s *TestSuite) TestStdout(c *C) {
1556         helperRecord := `{
1557                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1558                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1559                 "cwd": "/bin",
1560                 "environment": {"FROBIZ": "bilbo"},
1561                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1562                 "output_path": "/tmp",
1563                 "priority": 1,
1564                 "runtime_constraints": {},
1565                 "state": "Locked"
1566         }`
1567
1568         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1569                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1570         })
1571
1572         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1573         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1574         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1575 }
1576
1577 // Used by the TestStdoutWithWrongPath*()
1578 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1579         err := json.Unmarshal([]byte(record), &s.api.Container)
1580         c.Assert(err, IsNil)
1581         s.executor.runFunc = fn
1582         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1583         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1584                 return s.api, &KeepTestClient{}, nil, nil
1585         }
1586         return s.api, s.runner, s.runner.Run()
1587 }
1588
1589 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1590         _, _, err := s.stdoutErrorRunHelper(c, `{
1591     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1592     "output_path": "/tmp",
1593     "state": "Locked"
1594 }`, func() {})
1595         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1596 }
1597
1598 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1599         _, _, err := s.stdoutErrorRunHelper(c, `{
1600     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1601     "output_path": "/tmp",
1602     "state": "Locked"
1603 }`, func() {})
1604         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1605 }
1606
1607 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1608         _, _, err := s.stdoutErrorRunHelper(c, `{
1609     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1610     "output_path": "/tmp",
1611     "state": "Locked"
1612 }`, func() {})
1613         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1614 }
1615
1616 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1617         s.fullRunHelper(c, `{
1618     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1619     "container_image": "`+arvadostest.DockerImage112PDH+`",
1620     "cwd": "/bin",
1621     "environment": {},
1622     "mounts": {"/tmp": {"kind": "tmp"} },
1623     "output_path": "/tmp",
1624     "priority": 1,
1625     "runtime_constraints": {"API": true},
1626     "state": "Locked"
1627 }`, nil, 0, func() {
1628                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1629                 s.executor.exit <- 3
1630         })
1631         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1632         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1633         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1634 }
1635
1636 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1637         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1638         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1639         s.fullRunHelper(c, `{
1640     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1641     "container_image": "`+arvadostest.DockerImage112PDH+`",
1642     "cwd": "/bin",
1643     "environment": {},
1644     "mounts": {"/tmp": {"kind": "tmp"} },
1645     "output_path": "/tmp",
1646     "priority": 1,
1647     "runtime_constraints": {"API": true},
1648     "state": "Locked"
1649 }`, nil, 0, func() {
1650                 s.api.Container.Output = arvadostest.DockerImage112PDH
1651         })
1652
1653         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1654         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1655         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1656 }
1657
1658 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1659         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1660                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1661                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1662                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1663         }
1664         s.executor.runFunc = func() {
1665                 time.Sleep(time.Second)
1666                 s.executor.exit <- 137
1667         }
1668         record := `{
1669     "command": ["sleep", "1"],
1670     "container_image": "` + arvadostest.DockerImage112PDH + `",
1671     "cwd": "/bin",
1672     "environment": {},
1673     "mounts": {"/tmp": {"kind": "tmp"} },
1674     "output_path": "/tmp",
1675     "priority": 1,
1676     "runtime_constraints": {"API": true},
1677     "state": "Locked"
1678 }`
1679         err := json.Unmarshal([]byte(record), &s.api.Container)
1680         c.Assert(err, IsNil)
1681         err = s.runner.Run()
1682         c.Assert(err, IsNil)
1683         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1684         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1685         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1686         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1687         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1688 }
1689
1690 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1691         helperRecord := `{
1692                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1693                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1694                 "cwd": "/bin",
1695                 "environment": {"FROBIZ": "bilbo"},
1696                 "mounts": {
1697         "/tmp": {"kind": "tmp"},
1698         "/tmp/foo": {"kind": "collection",
1699                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1700                      "exclude_from_output": true
1701         },
1702         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1703     },
1704                 "output_path": "/tmp",
1705                 "priority": 1,
1706                 "runtime_constraints": {},
1707                 "state": "Locked"
1708         }`
1709
1710         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1711
1712         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1713                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1714         })
1715
1716         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1717         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1718         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1719 }
1720
1721 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1722         helperRecord := `{
1723                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1724                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1725                 "cwd": "/bin",
1726                 "environment": {"FROBIZ": "bilbo"},
1727                 "mounts": {
1728         "/tmp": {"kind": "tmp"},
1729         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1730         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1731         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1732         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1733         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1734     },
1735                 "output_path": "/tmp",
1736                 "priority": 1,
1737                 "runtime_constraints": {},
1738                 "state": "Locked",
1739                 "uuid": "zzzzz-dz642-202301130848001"
1740         }`
1741
1742         extraMounts := []string{
1743                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1744                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1745                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1746         }
1747
1748         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1749                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1750         })
1751
1752         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1753                 "/tmp":                   {realtemp + "/tmp1", false},
1754                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1755                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1756                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1757                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1758         })
1759
1760         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1761         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1762         output_count := uint(0)
1763         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1764                 if v["collection"] == nil {
1765                         continue
1766                 }
1767                 collection := v["collection"].(arvadosclient.Dict)
1768                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
1769                         continue
1770                 }
1771                 c.Check(v["ensure_unique_name"], Equals, true)
1772                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1773 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1774 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1775 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1776 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1777 `)
1778                 output_count++
1779         }
1780         c.Check(output_count, Not(Equals), uint(0))
1781 }
1782
1783 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1784         helperRecord := `{
1785                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1786                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1787                 "cwd": "/bin",
1788                 "environment": {"FROBIZ": "bilbo"},
1789                 "mounts": {
1790         "/tmp": {"kind": "tmp"},
1791         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1792         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1793     },
1794                 "output_path": "/tmp",
1795                 "priority": 1,
1796                 "runtime_constraints": {},
1797                 "state": "Locked",
1798                 "uuid": "zzzzz-dz642-202301130848002"
1799         }`
1800
1801         extraMounts := []string{
1802                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1803         }
1804
1805         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1806                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1807         })
1808
1809         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1810         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1811         output_count := uint(0)
1812         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1813                 if v["collection"] == nil {
1814                         continue
1815                 }
1816                 collection := v["collection"].(arvadosclient.Dict)
1817                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
1818                         continue
1819                 }
1820                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1821 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1822 `)
1823                 output_count++
1824         }
1825         c.Check(output_count, Not(Equals), uint(0))
1826 }
1827
1828 func (s *TestSuite) TestOutputError(c *C) {
1829         helperRecord := `{
1830                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1831                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1832                 "cwd": "/bin",
1833                 "environment": {"FROBIZ": "bilbo"},
1834                 "mounts": {
1835                         "/tmp": {"kind": "tmp"}
1836                 },
1837                 "output_path": "/tmp",
1838                 "priority": 1,
1839                 "runtime_constraints": {},
1840                 "state": "Locked"
1841         }`
1842         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1843                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1844         })
1845
1846         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1847 }
1848
1849 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1850         helperRecord := `{
1851                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1852                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1853                 "cwd": "/bin",
1854                 "environment": {"FROBIZ": "bilbo"},
1855                 "mounts": {
1856         "/tmp": {"kind": "tmp"},
1857         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1858         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1859     },
1860                 "output_path": "/tmp",
1861                 "priority": 1,
1862                 "runtime_constraints": {},
1863                 "state": "Locked"
1864         }`
1865
1866         extraMounts := []string{
1867                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1868         }
1869
1870         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1871                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1872         })
1873
1874         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1875         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1876         for _, v := range api.Content {
1877                 if v["collection"] != nil {
1878                         collection := v["collection"].(arvadosclient.Dict)
1879                         if strings.Index(collection["name"].(string), "output") == 0 {
1880                                 manifest := collection["manifest_text"].(string)
1881                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1882 `)
1883                         }
1884                 }
1885         }
1886 }
1887
1888 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1889         helperRecord := `{
1890                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1891                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1892                 "cwd": "/bin",
1893                 "environment": {"FROBIZ": "bilbo"},
1894                 "mounts": {
1895         "/tmp": {"kind": "tmp"},
1896         "stdin": {"kind": "json", "content": "foo"},
1897         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1898     },
1899                 "output_path": "/tmp",
1900                 "priority": 1,
1901                 "runtime_constraints": {},
1902                 "state": "Locked"
1903         }`
1904
1905         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1906                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1907         })
1908
1909         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1910         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1911         for _, v := range api.Content {
1912                 if v["collection"] != nil {
1913                         collection := v["collection"].(arvadosclient.Dict)
1914                         if strings.Index(collection["name"].(string), "output") == 0 {
1915                                 manifest := collection["manifest_text"].(string)
1916                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1917 `)
1918                         }
1919                 }
1920         }
1921 }
1922
1923 func (s *TestSuite) TestStderrMount(c *C) {
1924         api, cr, _ := s.fullRunHelper(c, `{
1925     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1926     "container_image": "`+arvadostest.DockerImage112PDH+`",
1927     "cwd": ".",
1928     "environment": {},
1929     "mounts": {"/tmp": {"kind": "tmp"},
1930                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1931                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1932     "output_path": "/tmp",
1933     "priority": 1,
1934     "runtime_constraints": {},
1935     "state": "Locked"
1936 }`, nil, 1, func() {
1937                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1938                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1939         })
1940
1941         final := api.CalledWith("container.state", "Complete")
1942         c.Assert(final, NotNil)
1943         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1944         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1945
1946         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1947 }
1948
1949 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1950         s.api.callraw = true
1951         err := s.runner.fetchContainerRecord()
1952         c.Assert(err, IsNil)
1953         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1954         c.Logf("%#v", s.runner.Container)
1955         c.Check(err, IsNil)
1956         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1957 }
1958
1959 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1960         nextState := ""
1961         for _, setup := range []func(){
1962                 func() {
1963                         c.Log("// waitErr = ocl runtime error")
1964                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1965                         nextState = "Cancelled"
1966                 },
1967                 func() {
1968                         c.Log("// loadErr = cannot connect")
1969                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1970                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1971                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1972                         c.Assert(err, IsNil)
1973                         nextState = "Queued"
1974                 },
1975         } {
1976                 s.SetUpTest(c)
1977                 setup()
1978                 s.fullRunHelper(c, `{
1979     "command": ["echo", "hello world"],
1980     "container_image": "`+arvadostest.DockerImage112PDH+`",
1981     "cwd": ".",
1982     "environment": {},
1983     "mounts": {"/tmp": {"kind": "tmp"} },
1984     "output_path": "/tmp",
1985     "priority": 1,
1986     "runtime_constraints": {},
1987     "state": "Locked"
1988 }`, nil, 0, func() {})
1989                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1990                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1991                 if s.runner.brokenNodeHook != "" {
1992                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1993                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1994                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1995                 } else {
1996                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1997                 }
1998         }
1999 }
2000
2001 func (s *TestSuite) TestBadCommand(c *C) {
2002         for _, startError := range []string{
2003                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
2004                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
2005                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2006         } {
2007                 s.SetUpTest(c)
2008                 s.executor.startErr = errors.New(startError)
2009                 s.fullRunHelper(c, `{
2010     "command": ["echo", "hello world"],
2011     "container_image": "`+arvadostest.DockerImage112PDH+`",
2012     "cwd": ".",
2013     "environment": {},
2014     "mounts": {"/tmp": {"kind": "tmp"} },
2015     "output_path": "/tmp",
2016     "priority": 1,
2017     "runtime_constraints": {},
2018     "state": "Locked"
2019 }`, nil, 0, func() {})
2020                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2021                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2022         }
2023 }
2024
2025 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2026         helperRecord := `{
2027                 "command": ["true"],
2028                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2029                 "cwd": "/bin",
2030                 "mounts": {
2031                     "/tmp": {"kind": "tmp"},
2032                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2033                 },
2034                 "secret_mounts": {
2035                 },
2036                 "output_path": "/tmp",
2037                 "priority": 1,
2038                 "runtime_constraints": {},
2039                 "state": "Locked"
2040         }`
2041
2042         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2043                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2044                 c.Check(err, IsNil)
2045                 c.Check(string(content), Equals, "mypassword")
2046         })
2047
2048         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2049         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2050         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2051         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2052
2053         // under secret mounts, not captured in output
2054         helperRecord = `{
2055                 "command": ["true"],
2056                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2057                 "cwd": "/bin",
2058                 "mounts": {
2059                     "/tmp": {"kind": "tmp"}
2060                 },
2061                 "secret_mounts": {
2062                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2063                 },
2064                 "output_path": "/tmp",
2065                 "priority": 1,
2066                 "runtime_constraints": {},
2067                 "state": "Locked"
2068         }`
2069
2070         s.SetUpTest(c)
2071         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2072                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2073                 c.Check(err, IsNil)
2074                 c.Check(string(content), Equals, "mypassword")
2075         })
2076
2077         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2078         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2079         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2080         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2081
2082         // under secret mounts, output dir is a collection, not captured in output
2083         helperRecord = `{
2084                 "command": ["true"],
2085                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2086                 "cwd": "/bin",
2087                 "mounts": {
2088                     "/tmp": {"kind": "collection", "writable": true}
2089                 },
2090                 "secret_mounts": {
2091                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2092                 },
2093                 "output_path": "/tmp",
2094                 "priority": 1,
2095                 "runtime_constraints": {},
2096                 "state": "Locked"
2097         }`
2098
2099         s.SetUpTest(c)
2100         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2101                 // secret.conf should be provisioned as a separate
2102                 // bind mount, i.e., it should not appear in the
2103                 // (fake) fuse filesystem as viewed from the host.
2104                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2105                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2106                         c.Logf("secret.conf: content %q, err %#v", content, err)
2107                 }
2108                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2109                 c.Check(err, IsNil)
2110         })
2111
2112         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2113         c.Check(err, IsNil)
2114         c.Check(string(content), Equals, "mypassword")
2115         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2116         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2117         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2118         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2119 }
2120
2121 func (s *TestSuite) TestCalculateCost(c *C) {
2122         defer func(s string) { lockdir = s }(lockdir)
2123         lockdir = c.MkDir()
2124         now := time.Now()
2125         cr := s.runner
2126         cr.costStartTime = now.Add(-time.Hour)
2127         var logbuf bytes.Buffer
2128         cr.CrunchLog.Immediate = log.New(&logbuf, "", 0)
2129
2130         // if there's no InstanceType env var, cost is calculated as 0
2131         os.Unsetenv("InstanceType")
2132         cost := cr.calculateCost(now)
2133         c.Check(cost, Equals, 0.0)
2134
2135         // with InstanceType env var and loadPrices() hasn't run (or
2136         // hasn't found any data), cost is calculated based on
2137         // InstanceType env var
2138         os.Setenv("InstanceType", `{"Price":1.2}`)
2139         defer os.Unsetenv("InstanceType")
2140         cost = cr.calculateCost(now)
2141         c.Check(cost, Equals, 1.2)
2142
2143         // first update tells us the spot price was $1/h until 30
2144         // minutes ago when it increased to $2/h
2145         j, err := json.Marshal([]cloud.InstancePrice{
2146                 {StartTime: now.Add(-4 * time.Hour), Price: 1.0},
2147                 {StartTime: now.Add(-time.Hour / 2), Price: 2.0},
2148         })
2149         c.Assert(err, IsNil)
2150         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2151         cr.loadPrices()
2152         cost = cr.calculateCost(now)
2153         c.Check(cost, Equals, 1.5)
2154
2155         // next update (via --list + SIGUSR2) tells us the spot price
2156         // increased to $3/h 15 minutes ago
2157         j, err = json.Marshal([]cloud.InstancePrice{
2158                 {StartTime: now.Add(-time.Hour / 3), Price: 2.0}, // dup of -time.Hour/2 price
2159                 {StartTime: now.Add(-time.Hour / 4), Price: 3.0},
2160         })
2161         c.Assert(err, IsNil)
2162         os.WriteFile(lockdir+"/"+pricesfile, j, 0777)
2163         cr.loadPrices()
2164         cost = cr.calculateCost(now)
2165         c.Check(cost, Equals, 1.0/2+2.0/4+3.0/4)
2166
2167         cost = cr.calculateCost(now.Add(-time.Hour / 2))
2168         c.Check(cost, Equals, 0.5)
2169
2170         c.Logf("%s", logbuf.String())
2171         c.Check(logbuf.String(), Matches, `(?ms).*Instance price changed to 1\.00 at 20.* changed to 2\.00 .* changed to 3\.00 .*`)
2172         c.Check(logbuf.String(), Not(Matches), `(?ms).*changed to 2\.00 .* changed to 2\.00 .*`)
2173 }
2174
2175 type FakeProcess struct {
2176         cmdLine []string
2177 }
2178
2179 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2180         return fp.cmdLine, nil
2181 }