lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/cmd"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/manifest"
  30         "golang.org/x/net/context"
  31
  32         . "gopkg.in/check.v1"
  33 )
  34
  35 // Gocheck boilerplate
  36 func TestCrunchExec(t *testing.T) {
  37         TestingT(t)
  38 }
  39
  40 var _ = Suite(&TestSuite{})
  41
  42 type TestSuite struct {
  43         client                   *arvados.Client
  44         api                      *ArvTestClient
  45         runner                   *ContainerRunner
  46         executor                 *stubExecutor
  47         keepmount                string
  48         keepmountTmp             []string
  49         testDispatcherKeepClient KeepTestClient
  50         testContainerKeepClient  KeepTestClient
  51 }
  52
  53 func (s *TestSuite) SetUpTest(c *C) {
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 133 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 134 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 135 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 136 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 137 func (e *stubExecutor) Close()                          { e.closed = true }
 138 func (e *stubExecutor) Wait(context.Context) (int, error) {
 139         return <-e.exit, e.waitErr
 140 }
 141 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 142         return nil, errors.New("unimplemented")
 143 }
 144 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 145
 146 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 147
 148 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 149 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 150 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 151
 152 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 153 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 154
 155 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 156 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 157 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 158 `
 159
 160 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 161
 162 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 163 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 164
 165 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 166 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 167
 168 func (client *ArvTestClient) Create(resourceType string,
 169         parameters arvadosclient.Dict,
 170         output interface{}) error {
 171
 172         client.Mutex.Lock()
 173         defer client.Mutex.Unlock()
 174
 175         client.Calls++
 176         client.Content = append(client.Content, parameters)
 177
 178         if resourceType == "logs" {
 179                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 180                 if client.Logs == nil {
 181                         client.Logs = make(map[string]*bytes.Buffer)
 182                 }
 183                 if client.Logs[et] == nil {
 184                         client.Logs[et] = &bytes.Buffer{}
 185                 }
 186                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 187         }
 188
 189         if resourceType == "collections" && output != nil {
 190                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 191                 md5sum := md5.Sum([]byte(mt))
 192                 outmap := output.(*arvados.Collection)
 193                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 194                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 195         }
 196
 197         return nil
 198 }
 199
 200 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 201         switch {
 202         case method == "GET" && resourceType == "containers" && action == "auth":
 203                 return json.Unmarshal([]byte(`{
 204                         "kind": "arvados#api_client_authorization",
 205                         "uuid": "`+fakeAuthUUID+`",
 206                         "api_token": "`+fakeAuthToken+`"
 207                         }`), output)
 208         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 209                 if client.secretMounts != nil {
 210                         return json.Unmarshal(client.secretMounts, output)
 211                 }
 212                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 213         default:
 214                 return fmt.Errorf("Not found")
 215         }
 216 }
 217
 218 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 219         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 220         var j []byte
 221         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 222                 j = []byte(`{
 223                         "kind": "arvados#nodeList",
 224                         "items": [{
 225                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 226                                 "hostname": "compute2",
 227                                 "properties": {"total_cpu_cores": 16}
 228                         }]}`)
 229         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 230                 if uuid == "" {
 231                         j, err = json.Marshal(map[string]interface{}{
 232                                 "items": []interface{}{client.Container},
 233                                 "kind":  "arvados#nodeList",
 234                         })
 235                 } else {
 236                         j, err = json.Marshal(client.Container)
 237                 }
 238         } else {
 239                 j = []byte(`{
 240                         "command": ["sleep", "1"],
 241                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 242                         "cwd": ".",
 243                         "environment": {},
 244                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 245                         "output_path": "/tmp",
 246                         "priority": 1,
 247                         "runtime_constraints": {}
 248                 }`)
 249         }
 250         return ioutil.NopCloser(bytes.NewReader(j)), err
 251 }
 252
 253 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 254         if resourceType == "collections" {
 255                 if uuid == hwPDH {
 256                         output.(*arvados.Collection).ManifestText = hwManifest
 257                 } else if uuid == otherPDH {
 258                         output.(*arvados.Collection).ManifestText = otherManifest
 259                 } else if uuid == normalizedWithSubdirsPDH {
 260                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 261                 } else if uuid == denormalizedWithSubdirsPDH {
 262                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 263                 }
 264         }
 265         if resourceType == "containers" {
 266                 (*output.(*arvados.Container)) = client.Container
 267         }
 268         return nil
 269 }
 270
 271 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 272         client.Mutex.Lock()
 273         defer client.Mutex.Unlock()
 274         client.Calls++
 275         client.Content = append(client.Content, parameters)
 276         if resourceType == "containers" {
 277                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 278                         client.WasSetRunning = true
 279                 }
 280         } else if resourceType == "collections" && output != nil {
 281                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 282                 output.(*arvados.Collection).UUID = uuid
 283                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 284         }
 285         return nil
 286 }
 287
 288 var discoveryMap = map[string]interface{}{
 289         "defaultTrashLifetime":               float64(1209600),
 290         "crunchLimitLogBytesPerJob":          float64(67108864),
 291         "crunchLogThrottleBytes":             float64(65536),
 292         "crunchLogThrottlePeriod":            float64(60),
 293         "crunchLogThrottleLines":             float64(1024),
 294         "crunchLogPartialLineThrottlePeriod": float64(5),
 295         "crunchLogBytesPerEvent":             float64(4096),
 296         "crunchLogSecondsBetweenEvents":      float64(1),
 297 }
 298
 299 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 300         return discoveryMap[key], nil
 301 }
 302
 303 // CalledWith returns the parameters from the first API call whose
 304 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 305 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 306 // no call matches, it returns nil.
 307 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 308 call:
 309         for _, content := range client.Content {
 310                 var v interface{} = content
 311                 for _, k := range strings.Split(jpath, ".") {
 312                         if dict, ok := v.(arvadosclient.Dict); !ok {
 313                                 continue call
 314                         } else {
 315                                 v = dict[k]
 316                         }
 317                 }
 318                 if v == expect {
 319                         return content
 320                 }
 321         }
 322         return nil
 323 }
 324
 325 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 326         return locator, nil
 327 }
 328
 329 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 330         client.Content = opts.Data
 331         return arvados.BlockWriteResponse{
 332                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 333         }, nil
 334 }
 335
 336 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 337         return 0, errors.New("not implemented")
 338 }
 339
 340 func (client *KeepTestClient) ClearBlockCache() {
 341 }
 342
 343 func (client *KeepTestClient) Close() {
 344         client.Content = nil
 345 }
 346
 347 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 348         client.StorageClasses = sc
 349 }
 350
 351 type FileWrapper struct {
 352         io.ReadCloser
 353         len int64
 354 }
 355
 356 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 357         return nil, errors.New("not implemented")
 358 }
 359
 360 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 361         return 0, errors.New("not implemented")
 362 }
 363
 364 func (fw FileWrapper) Size() int64 {
 365         return fw.len
 366 }
 367
 368 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 369         return nil, errors.New("not implemented")
 370 }
 371
 372 func (fw FileWrapper) Truncate(int64) error {
 373         return errors.New("not implemented")
 374 }
 375
 376 func (fw FileWrapper) Write([]byte) (int, error) {
 377         return 0, errors.New("not implemented")
 378 }
 379
 380 func (fw FileWrapper) Sync() error {
 381         return errors.New("not implemented")
 382 }
 383
 384 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 385         return nil, errors.New("not implemented")
 386 }
 387
 388 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 389         return errors.New("not implemented")
 390 }
 391
 392 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 393         if filename == hwImageID+".tar" {
 394                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 395                 client.Called = true
 396                 return FileWrapper{rdr, 1321984}, nil
 397         } else if filename == "/file1_in_main.txt" {
 398                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 399                 client.Called = true
 400                 return FileWrapper{rdr, 3}, nil
 401         }
 402         return nil, nil
 403 }
 404
 405 func (s *TestSuite) TestLoadImage(c *C) {
 406         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 407         s.runner.Container.Mounts = map[string]arvados.Mount{
 408                 "/out": {Kind: "tmp", Writable: true},
 409         }
 410         s.runner.Container.OutputPath = "/out"
 411
 412         _, err := s.runner.SetupMounts()
 413         c.Assert(err, IsNil)
 414
 415         imageID, err := s.runner.LoadImage()
 416         c.Check(err, IsNil)
 417         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 418         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 419
 420         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 421         s.executor.imageLoaded = false
 422         s.executor.loaded = ""
 423         s.executor.loadErr = errors.New("bork")
 424         imageID, err = s.runner.LoadImage()
 425         c.Check(err, ErrorMatches, ".*bork")
 426         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 427
 428         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 429         s.executor.imageLoaded = false
 430         s.executor.loaded = ""
 431         s.executor.loadErr = nil
 432         imageID, err = s.runner.LoadImage()
 433         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 434         c.Check(s.executor.loaded, Equals, "")
 435 }
 436
 437 type ArvErrorTestClient struct{}
 438
 439 func (ArvErrorTestClient) Create(resourceType string,
 440         parameters arvadosclient.Dict,
 441         output interface{}) error {
 442         return nil
 443 }
 444
 445 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 446         if method == "GET" && resourceType == "containers" && action == "auth" {
 447                 return nil
 448         }
 449         return errors.New("ArvError")
 450 }
 451
 452 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 453         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 454         return nil, errors.New("ArvError")
 455 }
 456
 457 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 458         return errors.New("ArvError")
 459 }
 460
 461 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 462         return nil
 463 }
 464
 465 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 466         return discoveryMap[key], nil
 467 }
 468
 469 type KeepErrorTestClient struct {
 470         KeepTestClient
 471 }
 472
 473 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 474         return nil, errors.New("KeepError")
 475 }
 476
 477 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 478         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 479 }
 480
 481 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 482         return "", errors.New("KeepError")
 483 }
 484
 485 type KeepReadErrorTestClient struct {
 486         KeepTestClient
 487 }
 488
 489 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 490         return 0, errors.New("KeepError")
 491 }
 492
 493 type ErrorReader struct {
 494         FileWrapper
 495 }
 496
 497 func (ErrorReader) Read(p []byte) (n int, err error) {
 498         return 0, errors.New("ErrorReader")
 499 }
 500
 501 func (ErrorReader) Seek(int64, int) (int64, error) {
 502         return 0, errors.New("ErrorReader")
 503 }
 504
 505 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 506         return ErrorReader{}, nil
 507 }
 508
 509 type ClosableBuffer struct {
 510         bytes.Buffer
 511 }
 512
 513 func (*ClosableBuffer) Close() error {
 514         return nil
 515 }
 516
 517 type TestLogs struct {
 518         Stdout ClosableBuffer
 519         Stderr ClosableBuffer
 520 }
 521
 522 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 523         if logstr == "stdout" {
 524                 return &tl.Stdout, nil
 525         }
 526         if logstr == "stderr" {
 527                 return &tl.Stderr, nil
 528         }
 529         return nil, errors.New("???")
 530 }
 531
 532 func dockerLog(fd byte, msg string) []byte {
 533         by := []byte(msg)
 534         header := make([]byte, 8+len(by))
 535         header[0] = fd
 536         header[7] = byte(len(by))
 537         copy(header[8:], by)
 538         return header
 539 }
 540
 541 func (s *TestSuite) TestRunContainer(c *C) {
 542         s.executor.runFunc = func() {
 543                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 544                 s.executor.exit <- 0
 545         }
 546
 547         var logs TestLogs
 548         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 549         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 550         s.runner.Container.Command = []string{"./hw"}
 551         s.runner.Container.OutputStorageClasses = []string{"default"}
 552
 553         imageID, err := s.runner.LoadImage()
 554         c.Assert(err, IsNil)
 555
 556         err = s.runner.CreateContainer(imageID, nil)
 557         c.Assert(err, IsNil)
 558
 559         err = s.runner.StartContainer()
 560         c.Assert(err, IsNil)
 561
 562         err = s.runner.WaitFinish()
 563         c.Assert(err, IsNil)
 564
 565         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 566         c.Check(logs.Stderr.String(), Equals, "")
 567 }
 568
 569 func (s *TestSuite) TestCommitLogs(c *C) {
 570         api := &ArvTestClient{}
 571         kc := &KeepTestClient{}
 572         defer kc.Close()
 573         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 574         c.Assert(err, IsNil)
 575         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 576
 577         cr.CrunchLog.Print("Hello world!")
 578         cr.CrunchLog.Print("Goodbye")
 579         cr.finalState = "Complete"
 580
 581         err = cr.CommitLogs()
 582         c.Check(err, IsNil)
 583
 584         c.Check(api.Calls, Equals, 2)
 585         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 586         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 587         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 588         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 589 }
 590
 591 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 592         api := &ArvTestClient{}
 593         kc := &KeepTestClient{}
 594         defer kc.Close()
 595         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 596         c.Assert(err, IsNil)
 597
 598         err = cr.UpdateContainerRunning()
 599         c.Check(err, IsNil)
 600
 601         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 602 }
 603
 604 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 605         api := &ArvTestClient{}
 606         kc := &KeepTestClient{}
 607         defer kc.Close()
 608         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 609         c.Assert(err, IsNil)
 610
 611         cr.LogsPDH = new(string)
 612         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 613
 614         cr.ExitCode = new(int)
 615         *cr.ExitCode = 42
 616         cr.finalState = "Complete"
 617
 618         err = cr.UpdateContainerFinal()
 619         c.Check(err, IsNil)
 620
 621         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 622         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 624 }
 625
 626 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 627         api := &ArvTestClient{}
 628         kc := &KeepTestClient{}
 629         defer kc.Close()
 630         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 631         c.Assert(err, IsNil)
 632         cr.cCancelled = true
 633         cr.finalState = "Cancelled"
 634
 635         err = cr.UpdateContainerFinal()
 636         c.Check(err, IsNil)
 637
 638         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 639         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 640         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 641 }
 642
 643 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 644 // dress rehearsal of the Run() function, starting from a JSON container record.
 645 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 646         err := json.Unmarshal([]byte(record), &s.api.Container)
 647         c.Assert(err, IsNil)
 648         initialState := s.api.Container.State
 649
 650         var sm struct {
 651                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 652         }
 653         err = json.Unmarshal([]byte(record), &sm)
 654         c.Check(err, IsNil)
 655         secretMounts, err := json.Marshal(sm)
 656         c.Assert(err, IsNil)
 657         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 658
 659         s.executor.runFunc = func() {
 660                 fn()
 661                 s.executor.exit <- exitCode
 662         }
 663
 664         s.runner.statInterval = 100 * time.Millisecond
 665         s.runner.containerWatchdogInterval = time.Second
 666
 667         realTemp := c.MkDir()
 668         tempcount := 0
 669         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 670                 tempcount++
 671                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 672                 err := os.Mkdir(d, os.ModePerm)
 673                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 674                         // Test case must have pre-populated the tempdir
 675                         err = nil
 676                 }
 677                 return d, err
 678         }
 679         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 680                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 681         }
 682
 683         if extraMounts != nil && len(extraMounts) > 0 {
 684                 err := s.runner.SetupArvMountPoint("keep")
 685                 c.Check(err, IsNil)
 686
 687                 for _, m := range extraMounts {
 688                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 689                 }
 690         }
 691
 692         err = s.runner.Run()
 693         if s.api.CalledWith("container.state", "Complete") != nil {
 694                 c.Check(err, IsNil)
 695         }
 696         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 697                 c.Check(s.api.WasSetRunning, Equals, true)
 698                 var lastupdate arvadosclient.Dict
 699                 for _, content := range s.api.Content {
 700                         if content["container"] != nil {
 701                                 lastupdate = content["container"].(arvadosclient.Dict)
 702                         }
 703                 }
 704                 if lastupdate["log"] == nil {
 705                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 706                 }
 707         }
 708
 709         if err != nil {
 710                 for k, v := range s.api.Logs {
 711                         c.Log(k)
 712                         c.Log(v.String())
 713                 }
 714         }
 715
 716         return s.api, s.runner, realTemp
 717 }
 718
 719 func (s *TestSuite) TestFullRunHello(c *C) {
 720         s.runner.enableMemoryLimit = true
 721         s.runner.networkMode = "default"
 722         s.fullRunHelper(c, `{
 723     "command": ["echo", "hello world"],
 724     "container_image": "`+arvadostest.DockerImage112PDH+`",
 725     "cwd": ".",
 726     "environment": {"foo":"bar","baz":"waz"},
 727     "mounts": {"/tmp": {"kind": "tmp"} },
 728     "output_path": "/tmp",
 729     "priority": 1,
 730     "runtime_constraints": {"vcpus":1,"ram":1000000},
 731     "state": "Locked",
 732     "output_storage_classes": ["default"]
 733 }`, nil, 0, func() {
 734                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 735                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 736                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 737                 c.Check(s.executor.created.VCPUs, Equals, 1)
 738                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 739                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 740                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 741                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 742                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 743         })
 744
 745         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 746         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 747         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 748         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 749         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 750 }
 751
 752 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 753         var ran bool
 754         s.fullRunHelper(c, `{
 755     "command": ["sleep", "3"],
 756     "container_image": "`+arvadostest.DockerImage112PDH+`",
 757     "cwd": ".",
 758     "environment": {},
 759     "mounts": {"/tmp": {"kind": "tmp"} },
 760     "output_path": "/tmp",
 761     "priority": 1,
 762     "runtime_constraints": {},
 763     "scheduling_parameters":{"max_run_time": 1},
 764     "state": "Running"
 765 }`, nil, 2, func() {
 766                 ran = true
 767         })
 768         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 769         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 770         c.Check(ran, Equals, false)
 771 }
 772
 773 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 774         s.fullRunHelper(c, `{
 775     "command": ["sleep", "3"],
 776     "container_image": "`+arvadostest.DockerImage112PDH+`",
 777     "cwd": ".",
 778     "environment": {},
 779     "mounts": {"/tmp": {"kind": "tmp"} },
 780     "output_path": "/tmp",
 781     "priority": 1,
 782     "runtime_constraints": {},
 783     "scheduling_parameters":{"max_run_time": 1},
 784     "state": "Locked"
 785 }`, nil, 0, func() {
 786                 time.Sleep(3 * time.Second)
 787         })
 788
 789         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 790         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 791 }
 792
 793 func (s *TestSuite) TestContainerWaitFails(c *C) {
 794         s.fullRunHelper(c, `{
 795     "command": ["sleep", "3"],
 796     "container_image": "`+arvadostest.DockerImage112PDH+`",
 797     "cwd": ".",
 798     "mounts": {"/tmp": {"kind": "tmp"} },
 799     "output_path": "/tmp",
 800     "priority": 1,
 801     "state": "Locked"
 802 }`, nil, 0, func() {
 803                 s.executor.waitErr = errors.New("Container is not running")
 804         })
 805
 806         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 807         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 808 }
 809
 810 func (s *TestSuite) TestCrunchstat(c *C) {
 811         s.fullRunHelper(c, `{
 812                 "command": ["sleep", "1"],
 813                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 814                 "cwd": ".",
 815                 "environment": {},
 816                 "mounts": {"/tmp": {"kind": "tmp"} },
 817                 "output_path": "/tmp",
 818                 "priority": 1,
 819                 "runtime_constraints": {},
 820                 "state": "Locked"
 821         }`, nil, 0, func() {
 822                 time.Sleep(time.Second)
 823         })
 824
 825         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 826         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 827
 828         // We didn't actually start a container, so crunchstat didn't
 829         // find accounting files and therefore didn't log any stats.
 830         // It should have logged a "can't find accounting files"
 831         // message after one poll interval, though, so we can confirm
 832         // it's alive:
 833         c.Assert(s.api.Logs["crunchstat"], NotNil)
 834         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 835
 836         // The "files never appeared" log assures us that we called
 837         // (*crunchstat.Reporter)Stop(), and that we set it up with
 838         // the correct container ID "abcde":
 839         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 840 }
 841
 842 func (s *TestSuite) TestNodeInfoLog(c *C) {
 843         os.Setenv("SLURMD_NODENAME", "compute2")
 844         s.fullRunHelper(c, `{
 845                 "command": ["sleep", "1"],
 846                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 847                 "cwd": ".",
 848                 "environment": {},
 849                 "mounts": {"/tmp": {"kind": "tmp"} },
 850                 "output_path": "/tmp",
 851                 "priority": 1,
 852                 "runtime_constraints": {},
 853                 "state": "Locked"
 854         }`, nil, 0,
 855                 func() {
 856                         time.Sleep(time.Second)
 857                 })
 858
 859         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 860         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 861
 862         c.Assert(s.api.Logs["node"], NotNil)
 863         json := s.api.Logs["node"].String()
 864         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 865         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 866         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 867
 868         c.Assert(s.api.Logs["node-info"], NotNil)
 869         json = s.api.Logs["node-info"].String()
 870         c.Check(json, Matches, `(?ms).*Host Information.*`)
 871         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 872         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 873         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 874         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 875 }
 876
 877 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 878         s.fullRunHelper(c, `{
 879                 "command": ["sleep", "1"],
 880                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 881                 "cwd": ".",
 882                 "environment": {},
 883                 "mounts": {"/tmp": {"kind": "tmp"} },
 884                 "output_path": "/tmp",
 885                 "priority": 1,
 886                 "runtime_constraints": {},
 887                 "state": "Locked"
 888         }`, nil, 0,
 889                 func() {
 890                 })
 891
 892         c.Assert(s.api.Logs["crunch-run"], NotNil)
 893         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 894         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 895         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 896         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 897 }
 898
 899 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
 900         var collection_create, container_update arvadosclient.Dict
 901         s.fullRunHelper(c, `{
 902                 "command": ["true"],
 903                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 904                 "cwd": ".",
 905                 "environment": {},
 906                 "mounts": {"/tmp": {"kind": "tmp"} },
 907                 "output_path": "/tmp",
 908                 "priority": 1,
 909                 "runtime_constraints": {},
 910                 "state": "Locked",
 911                 "uuid": "zzzzz-dz642-202301121543210"
 912         }`, nil, 0,
 913                 func() {
 914                         collection_create = s.api.CalledWith("ensure_unique_name", true)
 915                         container_update = s.api.CalledWith("container.state", "Running")
 916                 })
 917
 918         c.Assert(collection_create, NotNil)
 919         c.Check(collection_create["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-dz642-202301121543210")
 920         manifest_text := collection_create["collection"].(arvadosclient.Dict)["manifest_text"]
 921         // We check that the file size is at least two digits as an easy way to
 922         // check the file isn't empty.
 923         c.Check(manifest_text, Matches, `\. .* \d+:\d{2,}:node-info\.txt .*\n`)
 924         c.Check(manifest_text, Matches, `\. .* \d+:\d{2,}:node\.json .*\n`)
 925
 926         c.Assert(container_update, NotNil)
 927         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Matches, `zzzzz-4zz18-[0-9a-z]{15}`)
 928 }
 929
 930 func (s *TestSuite) TestContainerRecordLog(c *C) {
 931         s.fullRunHelper(c, `{
 932                 "command": ["sleep", "1"],
 933                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 934                 "cwd": ".",
 935                 "environment": {},
 936                 "mounts": {"/tmp": {"kind": "tmp"} },
 937                 "output_path": "/tmp",
 938                 "priority": 1,
 939                 "runtime_constraints": {},
 940                 "state": "Locked"
 941         }`, nil, 0,
 942                 func() {
 943                         time.Sleep(time.Second)
 944                 })
 945
 946         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 947         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 948
 949         c.Assert(s.api.Logs["container"], NotNil)
 950         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 951 }
 952
 953 func (s *TestSuite) TestFullRunStderr(c *C) {
 954         s.fullRunHelper(c, `{
 955     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 956     "container_image": "`+arvadostest.DockerImage112PDH+`",
 957     "cwd": ".",
 958     "environment": {},
 959     "mounts": {"/tmp": {"kind": "tmp"} },
 960     "output_path": "/tmp",
 961     "priority": 1,
 962     "runtime_constraints": {},
 963     "state": "Locked"
 964 }`, nil, 1, func() {
 965                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 966                 fmt.Fprintln(s.executor.created.Stderr, "world")
 967         })
 968
 969         final := s.api.CalledWith("container.state", "Complete")
 970         c.Assert(final, NotNil)
 971         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 972         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 973
 974         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 975         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 976 }
 977
 978 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 979         s.fullRunHelper(c, `{
 980     "command": ["pwd"],
 981     "container_image": "`+arvadostest.DockerImage112PDH+`",
 982     "cwd": ".",
 983     "environment": {},
 984     "mounts": {"/tmp": {"kind": "tmp"} },
 985     "output_path": "/tmp",
 986     "priority": 1,
 987     "runtime_constraints": {},
 988     "state": "Locked"
 989 }`, nil, 0, func() {
 990                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 991         })
 992
 993         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 994         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 995         c.Log(s.api.Logs["stdout"])
 996         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 997 }
 998
 999 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1000         s.fullRunHelper(c, `{
1001     "command": ["pwd"],
1002     "container_image": "`+arvadostest.DockerImage112PDH+`",
1003     "cwd": "/bin",
1004     "environment": {},
1005     "mounts": {"/tmp": {"kind": "tmp"} },
1006     "output_path": "/tmp",
1007     "priority": 1,
1008     "runtime_constraints": {},
1009     "state": "Locked"
1010 }`, nil, 0, func() {
1011                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1012         })
1013
1014         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1015         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1016         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1017 }
1018
1019 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1020         s.fullRunHelper(c, `{
1021     "command": ["pwd"],
1022     "container_image": "`+arvadostest.DockerImage112PDH+`",
1023     "cwd": "/bin",
1024     "environment": {},
1025     "mounts": {"/tmp": {"kind": "tmp"} },
1026     "output_path": "/tmp",
1027     "priority": 1,
1028     "runtime_constraints": {},
1029     "state": "Locked",
1030     "output_storage_classes": ["foo", "bar"]
1031 }`, nil, 0, func() {
1032                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1033         })
1034
1035         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1036         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1037         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1038         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1039         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1040 }
1041
1042 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1043         s.fullRunHelper(c, `{
1044     "command": ["pwd"],
1045     "container_image": "`+arvadostest.DockerImage112PDH+`",
1046     "cwd": "/bin",
1047     "environment": {},
1048     "mounts": {"/tmp": {"kind": "tmp"} },
1049     "output_path": "/tmp",
1050     "priority": 1,
1051     "runtime_constraints": {"cuda": {"device_count": 2}},
1052     "state": "Locked",
1053     "output_storage_classes": ["foo", "bar"]
1054 }`, nil, 0, func() {
1055                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1056         })
1057         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1058 }
1059
1060 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1061         s.fullRunHelper(c, `{
1062     "command": ["pwd"],
1063     "container_image": "`+arvadostest.DockerImage112PDH+`",
1064     "cwd": "/bin",
1065     "environment": {},
1066     "mounts": {"/tmp": {"kind": "tmp"} },
1067     "output_path": "/tmp",
1068     "priority": 1,
1069     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1070     "state": "Locked",
1071     "output_storage_classes": ["foo", "bar"]
1072 }`, nil, 0, func() {
1073                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1074         })
1075         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1076 }
1077
1078 func (s *TestSuite) TestStopOnSignal(c *C) {
1079         s.executor.runFunc = func() {
1080                 s.executor.created.Stdout.Write([]byte("foo\n"))
1081                 s.runner.SigChan <- syscall.SIGINT
1082         }
1083         s.testStopContainer(c)
1084 }
1085
1086 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1087         s.executor.runFunc = func() {
1088                 s.executor.created.Stdout.Write([]byte("foo\n"))
1089                 s.runner.ArvMountExit <- nil
1090                 close(s.runner.ArvMountExit)
1091         }
1092         s.runner.ArvMountExit = make(chan error)
1093         s.testStopContainer(c)
1094 }
1095
1096 func (s *TestSuite) testStopContainer(c *C) {
1097         record := `{
1098     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1099     "container_image": "` + arvadostest.DockerImage112PDH + `",
1100     "cwd": ".",
1101     "environment": {},
1102     "mounts": {"/tmp": {"kind": "tmp"} },
1103     "output_path": "/tmp",
1104     "priority": 1,
1105     "runtime_constraints": {},
1106     "state": "Locked"
1107 }`
1108
1109         err := json.Unmarshal([]byte(record), &s.api.Container)
1110         c.Assert(err, IsNil)
1111
1112         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1113         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1114                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1115         }
1116
1117         done := make(chan error)
1118         go func() {
1119                 done <- s.runner.Run()
1120         }()
1121         select {
1122         case <-time.After(20 * time.Second):
1123                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1124                 c.Fatal("timed out")
1125         case err = <-done:
1126                 c.Check(err, IsNil)
1127         }
1128         for k, v := range s.api.Logs {
1129                 c.Log(k)
1130                 c.Log(v.String(), "\n")
1131         }
1132
1133         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1134         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1135         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1136 }
1137
1138 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1139         s.fullRunHelper(c, `{
1140     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1141     "container_image": "`+arvadostest.DockerImage112PDH+`",
1142     "cwd": "/bin",
1143     "environment": {"FROBIZ": "bilbo"},
1144     "mounts": {"/tmp": {"kind": "tmp"} },
1145     "output_path": "/tmp",
1146     "priority": 1,
1147     "runtime_constraints": {},
1148     "state": "Locked"
1149 }`, nil, 0, func() {
1150                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1151         })
1152
1153         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1154         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1155         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1156 }
1157
1158 type ArvMountCmdLine struct {
1159         Cmd   []string
1160         token string
1161 }
1162
1163 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1164         am.Cmd = c
1165         am.token = token
1166         return nil, nil
1167 }
1168
1169 func stubCert(temp string) string {
1170         path := temp + "/ca-certificates.crt"
1171         crt, _ := os.Create(path)
1172         crt.Close()
1173         arvadosclient.CertFiles = []string{path}
1174         return path
1175 }
1176
1177 func (s *TestSuite) TestSetupMounts(c *C) {
1178         cr := s.runner
1179         am := &ArvMountCmdLine{}
1180         cr.RunArvMount = am.ArvMountTest
1181         cr.ContainerArvClient = &ArvTestClient{}
1182         cr.ContainerKeepClient = &KeepTestClient{}
1183         cr.Container.OutputStorageClasses = []string{"default"}
1184
1185         realTemp := c.MkDir()
1186         certTemp := c.MkDir()
1187         stubCertPath := stubCert(certTemp)
1188         cr.parentTemp = realTemp
1189
1190         i := 0
1191         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1192                 i++
1193                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1194                 err := os.Mkdir(d, os.ModePerm)
1195                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1196                         // Test case must have pre-populated the tempdir
1197                         err = nil
1198                 }
1199                 return d, err
1200         }
1201
1202         checkEmpty := func() {
1203                 // Should be deleted.
1204                 _, err := os.Stat(realTemp)
1205                 c.Assert(os.IsNotExist(err), Equals, true)
1206
1207                 // Now recreate it for the next test.
1208                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1209         }
1210
1211         {
1212                 i = 0
1213                 cr.ArvMountPoint = ""
1214                 cr.Container.Mounts = make(map[string]arvados.Mount)
1215                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1216                 cr.Container.OutputPath = "/tmp"
1217                 cr.statInterval = 5 * time.Second
1218                 bindmounts, err := cr.SetupMounts()
1219                 c.Check(err, IsNil)
1220                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1221                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1222                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1223                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1224                 os.RemoveAll(cr.ArvMountPoint)
1225                 cr.CleanupDirs()
1226                 checkEmpty()
1227         }
1228
1229         {
1230                 i = 0
1231                 cr.ArvMountPoint = ""
1232                 cr.Container.Mounts = make(map[string]arvados.Mount)
1233                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1234                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1235                 cr.Container.OutputPath = "/out"
1236                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1237
1238                 bindmounts, err := cr.SetupMounts()
1239                 c.Check(err, IsNil)
1240                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1241                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1242                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1243                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1244                 os.RemoveAll(cr.ArvMountPoint)
1245                 cr.CleanupDirs()
1246                 checkEmpty()
1247         }
1248
1249         {
1250                 i = 0
1251                 cr.ArvMountPoint = ""
1252                 cr.Container.Mounts = make(map[string]arvados.Mount)
1253                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1254                 cr.Container.OutputPath = "/tmp"
1255                 cr.Container.RuntimeConstraints.API = true
1256                 cr.Container.OutputStorageClasses = []string{"default"}
1257
1258                 bindmounts, err := cr.SetupMounts()
1259                 c.Check(err, IsNil)
1260                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1261                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1262                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1263                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1264                 os.RemoveAll(cr.ArvMountPoint)
1265                 cr.CleanupDirs()
1266                 checkEmpty()
1267
1268                 cr.Container.RuntimeConstraints.API = false
1269         }
1270
1271         {
1272                 i = 0
1273                 cr.ArvMountPoint = ""
1274                 cr.Container.Mounts = map[string]arvados.Mount{
1275                         "/keeptmp": {Kind: "collection", Writable: true},
1276                 }
1277                 cr.Container.OutputPath = "/keeptmp"
1278
1279                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1280
1281                 bindmounts, err := cr.SetupMounts()
1282                 c.Check(err, IsNil)
1283                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1284                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1285                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1286                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1287                 os.RemoveAll(cr.ArvMountPoint)
1288                 cr.CleanupDirs()
1289                 checkEmpty()
1290         }
1291
1292         {
1293                 i = 0
1294                 cr.ArvMountPoint = ""
1295                 cr.Container.Mounts = map[string]arvados.Mount{
1296                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1297                         "/keepout": {Kind: "collection", Writable: true},
1298                 }
1299                 cr.Container.OutputPath = "/keepout"
1300
1301                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1302                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1303
1304                 bindmounts, err := cr.SetupMounts()
1305                 c.Check(err, IsNil)
1306                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1307                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1308                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1309                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1310                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1311                         "/keepout": {realTemp + "/keep1/tmp0", false},
1312                 })
1313                 os.RemoveAll(cr.ArvMountPoint)
1314                 cr.CleanupDirs()
1315                 checkEmpty()
1316         }
1317
1318         {
1319                 i = 0
1320                 cr.ArvMountPoint = ""
1321                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1322                 cr.Container.Mounts = map[string]arvados.Mount{
1323                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1324                         "/keepout": {Kind: "collection", Writable: true},
1325                 }
1326                 cr.Container.OutputPath = "/keepout"
1327
1328                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1329                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1330
1331                 bindmounts, err := cr.SetupMounts()
1332                 c.Check(err, IsNil)
1333                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1334                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1335                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1336                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1337                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1338                         "/keepout": {realTemp + "/keep1/tmp0", false},
1339                 })
1340                 os.RemoveAll(cr.ArvMountPoint)
1341                 cr.CleanupDirs()
1342                 checkEmpty()
1343         }
1344
1345         for _, test := range []struct {
1346                 in  interface{}
1347                 out string
1348         }{
1349                 {in: "foo", out: `"foo"`},
1350                 {in: nil, out: `null`},
1351                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1352         } {
1353                 i = 0
1354                 cr.ArvMountPoint = ""
1355                 cr.Container.Mounts = map[string]arvados.Mount{
1356                         "/mnt/test.json": {Kind: "json", Content: test.in},
1357                 }
1358                 bindmounts, err := cr.SetupMounts()
1359                 c.Check(err, IsNil)
1360                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1361                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1362                 })
1363                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1364                 c.Check(err, IsNil)
1365                 c.Check(content, DeepEquals, []byte(test.out))
1366                 os.RemoveAll(cr.ArvMountPoint)
1367                 cr.CleanupDirs()
1368                 checkEmpty()
1369         }
1370
1371         for _, test := range []struct {
1372                 in  interface{}
1373                 out string
1374         }{
1375                 {in: "foo", out: `foo`},
1376                 {in: nil, out: "error"},
1377                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1378         } {
1379                 i = 0
1380                 cr.ArvMountPoint = ""
1381                 cr.Container.Mounts = map[string]arvados.Mount{
1382                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1383                 }
1384                 bindmounts, err := cr.SetupMounts()
1385                 if test.out == "error" {
1386                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1387                 } else {
1388                         c.Check(err, IsNil)
1389                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1390                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1391                         })
1392                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1393                         c.Check(err, IsNil)
1394                         c.Check(content, DeepEquals, []byte(test.out))
1395                 }
1396                 os.RemoveAll(cr.ArvMountPoint)
1397                 cr.CleanupDirs()
1398                 checkEmpty()
1399         }
1400
1401         // Read-only mount points are allowed underneath output_dir mount point
1402         {
1403                 i = 0
1404                 cr.ArvMountPoint = ""
1405                 cr.Container.Mounts = make(map[string]arvados.Mount)
1406                 cr.Container.Mounts = map[string]arvados.Mount{
1407                         "/tmp":     {Kind: "tmp"},
1408                         "/tmp/foo": {Kind: "collection"},
1409                 }
1410                 cr.Container.OutputPath = "/tmp"
1411
1412                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1413
1414                 bindmounts, err := cr.SetupMounts()
1415                 c.Check(err, IsNil)
1416                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1417                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1418                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1419                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1420                         "/tmp":     {realTemp + "/tmp2", false},
1421                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1422                 })
1423                 os.RemoveAll(cr.ArvMountPoint)
1424                 cr.CleanupDirs()
1425                 checkEmpty()
1426         }
1427
1428         // Writable mount points copied to output_dir mount point
1429         {
1430                 i = 0
1431                 cr.ArvMountPoint = ""
1432                 cr.Container.Mounts = make(map[string]arvados.Mount)
1433                 cr.Container.Mounts = map[string]arvados.Mount{
1434                         "/tmp": {Kind: "tmp"},
1435                         "/tmp/foo": {Kind: "collection",
1436                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1437                                 Writable:         true},
1438                         "/tmp/bar": {Kind: "collection",
1439                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1440                                 Path:             "baz",
1441                                 Writable:         true},
1442                 }
1443                 cr.Container.OutputPath = "/tmp"
1444
1445                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1446                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1447
1448                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1449                 rf.Write([]byte("bar"))
1450                 rf.Close()
1451
1452                 _, err := cr.SetupMounts()
1453                 c.Check(err, IsNil)
1454                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1455                 c.Check(err, IsNil)
1456                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1457                 c.Check(err, IsNil)
1458                 os.RemoveAll(cr.ArvMountPoint)
1459                 cr.CleanupDirs()
1460                 checkEmpty()
1461         }
1462
1463         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1464         {
1465                 i = 0
1466                 cr.ArvMountPoint = ""
1467                 cr.Container.Mounts = make(map[string]arvados.Mount)
1468                 cr.Container.Mounts = map[string]arvados.Mount{
1469                         "/tmp":     {Kind: "tmp"},
1470                         "/tmp/foo": {Kind: "tmp"},
1471                 }
1472                 cr.Container.OutputPath = "/tmp"
1473
1474                 _, err := cr.SetupMounts()
1475                 c.Check(err, NotNil)
1476                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1477                 os.RemoveAll(cr.ArvMountPoint)
1478                 cr.CleanupDirs()
1479                 checkEmpty()
1480         }
1481
1482         // Only mount point of kind 'collection' is allowed for stdin
1483         {
1484                 i = 0
1485                 cr.ArvMountPoint = ""
1486                 cr.Container.Mounts = make(map[string]arvados.Mount)
1487                 cr.Container.Mounts = map[string]arvados.Mount{
1488                         "stdin": {Kind: "tmp"},
1489                 }
1490
1491                 _, err := cr.SetupMounts()
1492                 c.Check(err, NotNil)
1493                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1494                 os.RemoveAll(cr.ArvMountPoint)
1495                 cr.CleanupDirs()
1496                 checkEmpty()
1497         }
1498
1499         // git_tree mounts
1500         {
1501                 i = 0
1502                 cr.ArvMountPoint = ""
1503                 (*GitMountSuite)(nil).useTestGitServer(c)
1504                 cr.token = arvadostest.ActiveToken
1505                 cr.Container.Mounts = make(map[string]arvados.Mount)
1506                 cr.Container.Mounts = map[string]arvados.Mount{
1507                         "/tip": {
1508                                 Kind:   "git_tree",
1509                                 UUID:   arvadostest.Repository2UUID,
1510                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1511                                 Path:   "/",
1512                         },
1513                         "/non-tip": {
1514                                 Kind:   "git_tree",
1515                                 UUID:   arvadostest.Repository2UUID,
1516                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1517                                 Path:   "/",
1518                         },
1519                 }
1520                 cr.Container.OutputPath = "/tmp"
1521
1522                 bindmounts, err := cr.SetupMounts()
1523                 c.Check(err, IsNil)
1524
1525                 for path, mount := range bindmounts {
1526                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1527                 }
1528
1529                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1530                 c.Check(err, IsNil)
1531                 c.Check(string(data), Equals, "\000\001\002\003")
1532                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1533                 c.Check(err, FitsTypeOf, &os.PathError{})
1534                 c.Check(os.IsNotExist(err), Equals, true)
1535
1536                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1537                 c.Check(err, IsNil)
1538                 c.Check(string(data), Equals, "\000\001\002\003")
1539                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1540                 c.Check(err, IsNil)
1541                 c.Check(string(data), Equals, "testfile\n")
1542
1543                 cr.CleanupDirs()
1544                 checkEmpty()
1545         }
1546 }
1547
1548 func (s *TestSuite) TestStdout(c *C) {
1549         helperRecord := `{
1550                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1551                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1552                 "cwd": "/bin",
1553                 "environment": {"FROBIZ": "bilbo"},
1554                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1555                 "output_path": "/tmp",
1556                 "priority": 1,
1557                 "runtime_constraints": {},
1558                 "state": "Locked"
1559         }`
1560
1561         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1562                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1563         })
1564
1565         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1566         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1567         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1568 }
1569
1570 // Used by the TestStdoutWithWrongPath*()
1571 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1572         err := json.Unmarshal([]byte(record), &s.api.Container)
1573         c.Assert(err, IsNil)
1574         s.executor.runFunc = fn
1575         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1576         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1577                 return s.api, &KeepTestClient{}, nil, nil
1578         }
1579         return s.api, s.runner, s.runner.Run()
1580 }
1581
1582 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1583         _, _, err := s.stdoutErrorRunHelper(c, `{
1584     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1585     "output_path": "/tmp",
1586     "state": "Locked"
1587 }`, func() {})
1588         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1589 }
1590
1591 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1592         _, _, err := s.stdoutErrorRunHelper(c, `{
1593     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1594     "output_path": "/tmp",
1595     "state": "Locked"
1596 }`, func() {})
1597         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1598 }
1599
1600 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1601         _, _, err := s.stdoutErrorRunHelper(c, `{
1602     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1603     "output_path": "/tmp",
1604     "state": "Locked"
1605 }`, func() {})
1606         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1607 }
1608
1609 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1610         s.fullRunHelper(c, `{
1611     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1612     "container_image": "`+arvadostest.DockerImage112PDH+`",
1613     "cwd": "/bin",
1614     "environment": {},
1615     "mounts": {"/tmp": {"kind": "tmp"} },
1616     "output_path": "/tmp",
1617     "priority": 1,
1618     "runtime_constraints": {"API": true},
1619     "state": "Locked"
1620 }`, nil, 0, func() {
1621                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1622                 s.executor.exit <- 3
1623         })
1624         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1625         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1626         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1627 }
1628
1629 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1630         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1631         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1632         s.fullRunHelper(c, `{
1633     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1634     "container_image": "`+arvadostest.DockerImage112PDH+`",
1635     "cwd": "/bin",
1636     "environment": {},
1637     "mounts": {"/tmp": {"kind": "tmp"} },
1638     "output_path": "/tmp",
1639     "priority": 1,
1640     "runtime_constraints": {"API": true},
1641     "state": "Locked"
1642 }`, nil, 0, func() {
1643                 s.api.Container.Output = arvadostest.DockerImage112PDH
1644         })
1645
1646         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1647         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1648         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1649 }
1650
1651 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1652         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1653                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1654                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1655                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1656         }
1657         s.executor.runFunc = func() {
1658                 time.Sleep(time.Second)
1659                 s.executor.exit <- 137
1660         }
1661         record := `{
1662     "command": ["sleep", "1"],
1663     "container_image": "` + arvadostest.DockerImage112PDH + `",
1664     "cwd": "/bin",
1665     "environment": {},
1666     "mounts": {"/tmp": {"kind": "tmp"} },
1667     "output_path": "/tmp",
1668     "priority": 1,
1669     "runtime_constraints": {"API": true},
1670     "state": "Locked"
1671 }`
1672         err := json.Unmarshal([]byte(record), &s.api.Container)
1673         c.Assert(err, IsNil)
1674         err = s.runner.Run()
1675         c.Assert(err, IsNil)
1676         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1677         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1678         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1679         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1680         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1681 }
1682
1683 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1684         helperRecord := `{
1685                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1686                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1687                 "cwd": "/bin",
1688                 "environment": {"FROBIZ": "bilbo"},
1689                 "mounts": {
1690         "/tmp": {"kind": "tmp"},
1691         "/tmp/foo": {"kind": "collection",
1692                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1693                      "exclude_from_output": true
1694         },
1695         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1696     },
1697                 "output_path": "/tmp",
1698                 "priority": 1,
1699                 "runtime_constraints": {},
1700                 "state": "Locked"
1701         }`
1702
1703         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1704
1705         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1706                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1707         })
1708
1709         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1710         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1711         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1712 }
1713
1714 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1715         helperRecord := `{
1716                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1717                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1718                 "cwd": "/bin",
1719                 "environment": {"FROBIZ": "bilbo"},
1720                 "mounts": {
1721         "/tmp": {"kind": "tmp"},
1722         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1723         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1724         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1725         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1726         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1727     },
1728                 "output_path": "/tmp",
1729                 "priority": 1,
1730                 "runtime_constraints": {},
1731                 "state": "Locked",
1732                 "uuid": "zzzzz-dz642-202301130848001"
1733         }`
1734
1735         extraMounts := []string{
1736                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1737                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1738                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1739         }
1740
1741         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1742                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1743         })
1744
1745         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1746                 "/tmp":                   {realtemp + "/tmp1", false},
1747                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1748                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1749                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1750                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1751         })
1752
1753         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1754         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1755         output_count := uint(0)
1756         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1757                 if v["collection"] == nil {
1758                         continue
1759                 }
1760                 collection := v["collection"].(arvadosclient.Dict)
1761                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
1762                         continue
1763                 }
1764                 c.Check(v["ensure_unique_name"], Equals, true)
1765                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1766 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1767 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1768 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1769 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1770 `)
1771                 output_count++
1772         }
1773         c.Check(output_count, Not(Equals), uint(0))
1774 }
1775
1776 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1777         helperRecord := `{
1778                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1779                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1780                 "cwd": "/bin",
1781                 "environment": {"FROBIZ": "bilbo"},
1782                 "mounts": {
1783         "/tmp": {"kind": "tmp"},
1784         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1785         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1786     },
1787                 "output_path": "/tmp",
1788                 "priority": 1,
1789                 "runtime_constraints": {},
1790                 "state": "Locked",
1791                 "uuid": "zzzzz-dz642-202301130848002"
1792         }`
1793
1794         extraMounts := []string{
1795                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1796         }
1797
1798         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1799                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1800         })
1801
1802         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1803         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1804         output_count := uint(0)
1805         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1806                 if v["collection"] == nil {
1807                         continue
1808                 }
1809                 collection := v["collection"].(arvadosclient.Dict)
1810                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
1811                         continue
1812                 }
1813                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1814 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1815 `)
1816                 output_count++
1817         }
1818         c.Check(output_count, Not(Equals), uint(0))
1819 }
1820
1821 func (s *TestSuite) TestOutputError(c *C) {
1822         helperRecord := `{
1823                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1824                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1825                 "cwd": "/bin",
1826                 "environment": {"FROBIZ": "bilbo"},
1827                 "mounts": {
1828                         "/tmp": {"kind": "tmp"}
1829                 },
1830                 "output_path": "/tmp",
1831                 "priority": 1,
1832                 "runtime_constraints": {},
1833                 "state": "Locked"
1834         }`
1835         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1836                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1837         })
1838
1839         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1840 }
1841
1842 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1843         helperRecord := `{
1844                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1845                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1846                 "cwd": "/bin",
1847                 "environment": {"FROBIZ": "bilbo"},
1848                 "mounts": {
1849         "/tmp": {"kind": "tmp"},
1850         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1851         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1852     },
1853                 "output_path": "/tmp",
1854                 "priority": 1,
1855                 "runtime_constraints": {},
1856                 "state": "Locked"
1857         }`
1858
1859         extraMounts := []string{
1860                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1861         }
1862
1863         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1864                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1865         })
1866
1867         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1868         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1869         for _, v := range api.Content {
1870                 if v["collection"] != nil {
1871                         collection := v["collection"].(arvadosclient.Dict)
1872                         if strings.Index(collection["name"].(string), "output") == 0 {
1873                                 manifest := collection["manifest_text"].(string)
1874                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1875 `)
1876                         }
1877                 }
1878         }
1879 }
1880
1881 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1882         helperRecord := `{
1883                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1884                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1885                 "cwd": "/bin",
1886                 "environment": {"FROBIZ": "bilbo"},
1887                 "mounts": {
1888         "/tmp": {"kind": "tmp"},
1889         "stdin": {"kind": "json", "content": "foo"},
1890         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1891     },
1892                 "output_path": "/tmp",
1893                 "priority": 1,
1894                 "runtime_constraints": {},
1895                 "state": "Locked"
1896         }`
1897
1898         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1899                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1900         })
1901
1902         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1903         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1904         for _, v := range api.Content {
1905                 if v["collection"] != nil {
1906                         collection := v["collection"].(arvadosclient.Dict)
1907                         if strings.Index(collection["name"].(string), "output") == 0 {
1908                                 manifest := collection["manifest_text"].(string)
1909                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1910 `)
1911                         }
1912                 }
1913         }
1914 }
1915
1916 func (s *TestSuite) TestStderrMount(c *C) {
1917         api, cr, _ := s.fullRunHelper(c, `{
1918     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1919     "container_image": "`+arvadostest.DockerImage112PDH+`",
1920     "cwd": ".",
1921     "environment": {},
1922     "mounts": {"/tmp": {"kind": "tmp"},
1923                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1924                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1925     "output_path": "/tmp",
1926     "priority": 1,
1927     "runtime_constraints": {},
1928     "state": "Locked"
1929 }`, nil, 1, func() {
1930                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1931                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1932         })
1933
1934         final := api.CalledWith("container.state", "Complete")
1935         c.Assert(final, NotNil)
1936         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1937         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1938
1939         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1940 }
1941
1942 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1943         s.api.callraw = true
1944         err := s.runner.fetchContainerRecord()
1945         c.Assert(err, IsNil)
1946         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1947         c.Logf("%#v", s.runner.Container)
1948         c.Check(err, IsNil)
1949         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1950 }
1951
1952 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1953         nextState := ""
1954         for _, setup := range []func(){
1955                 func() {
1956                         c.Log("// waitErr = ocl runtime error")
1957                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1958                         nextState = "Cancelled"
1959                 },
1960                 func() {
1961                         c.Log("// loadErr = cannot connect")
1962                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1963                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1964                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1965                         c.Assert(err, IsNil)
1966                         nextState = "Queued"
1967                 },
1968         } {
1969                 s.SetUpTest(c)
1970                 setup()
1971                 s.fullRunHelper(c, `{
1972     "command": ["echo", "hello world"],
1973     "container_image": "`+arvadostest.DockerImage112PDH+`",
1974     "cwd": ".",
1975     "environment": {},
1976     "mounts": {"/tmp": {"kind": "tmp"} },
1977     "output_path": "/tmp",
1978     "priority": 1,
1979     "runtime_constraints": {},
1980     "state": "Locked"
1981 }`, nil, 0, func() {})
1982                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1983                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1984                 if s.runner.brokenNodeHook != "" {
1985                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1986                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1987                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1988                 } else {
1989                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1990                 }
1991         }
1992 }
1993
1994 func (s *TestSuite) TestBadCommand(c *C) {
1995         for _, startError := range []string{
1996                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1997                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1998                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1999         } {
2000                 s.SetUpTest(c)
2001                 s.executor.startErr = errors.New(startError)
2002                 s.fullRunHelper(c, `{
2003     "command": ["echo", "hello world"],
2004     "container_image": "`+arvadostest.DockerImage112PDH+`",
2005     "cwd": ".",
2006     "environment": {},
2007     "mounts": {"/tmp": {"kind": "tmp"} },
2008     "output_path": "/tmp",
2009     "priority": 1,
2010     "runtime_constraints": {},
2011     "state": "Locked"
2012 }`, nil, 0, func() {})
2013                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2014                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2015         }
2016 }
2017
2018 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2019         helperRecord := `{
2020                 "command": ["true"],
2021                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2022                 "cwd": "/bin",
2023                 "mounts": {
2024                     "/tmp": {"kind": "tmp"},
2025                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2026                 },
2027                 "secret_mounts": {
2028                 },
2029                 "output_path": "/tmp",
2030                 "priority": 1,
2031                 "runtime_constraints": {},
2032                 "state": "Locked"
2033         }`
2034
2035         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2036                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2037                 c.Check(err, IsNil)
2038                 c.Check(string(content), Equals, "mypassword")
2039         })
2040
2041         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2042         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2043         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2044         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2045
2046         // under secret mounts, not captured in output
2047         helperRecord = `{
2048                 "command": ["true"],
2049                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2050                 "cwd": "/bin",
2051                 "mounts": {
2052                     "/tmp": {"kind": "tmp"}
2053                 },
2054                 "secret_mounts": {
2055                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2056                 },
2057                 "output_path": "/tmp",
2058                 "priority": 1,
2059                 "runtime_constraints": {},
2060                 "state": "Locked"
2061         }`
2062
2063         s.SetUpTest(c)
2064         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2065                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2066                 c.Check(err, IsNil)
2067                 c.Check(string(content), Equals, "mypassword")
2068         })
2069
2070         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2071         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2072         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2073         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2074
2075         // under secret mounts, output dir is a collection, not captured in output
2076         helperRecord = `{
2077                 "command": ["true"],
2078                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2079                 "cwd": "/bin",
2080                 "mounts": {
2081                     "/tmp": {"kind": "collection", "writable": true}
2082                 },
2083                 "secret_mounts": {
2084                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2085                 },
2086                 "output_path": "/tmp",
2087                 "priority": 1,
2088                 "runtime_constraints": {},
2089                 "state": "Locked"
2090         }`
2091
2092         s.SetUpTest(c)
2093         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2094                 // secret.conf should be provisioned as a separate
2095                 // bind mount, i.e., it should not appear in the
2096                 // (fake) fuse filesystem as viewed from the host.
2097                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2098                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2099                         c.Logf("secret.conf: content %q, err %#v", content, err)
2100                 }
2101                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2102                 c.Check(err, IsNil)
2103         })
2104
2105         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2106         c.Check(err, IsNil)
2107         c.Check(string(content), Equals, "mypassword")
2108         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2109         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2110         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2111         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2112 }
2113
2114 type FakeProcess struct {
2115         cmdLine []string
2116 }
2117
2118 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2119         return fp.cmdLine, nil
2120 }