lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/cmd"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/manifest"
  30         "golang.org/x/net/context"
  31
  32         . "gopkg.in/check.v1"
  33 )
  34
  35 // Gocheck boilerplate
  36 func TestCrunchExec(t *testing.T) {
  37         TestingT(t)
  38 }
  39
  40 var _ = Suite(&TestSuite{})
  41
  42 type TestSuite struct {
  43         client                   *arvados.Client
  44         api                      *ArvTestClient
  45         runner                   *ContainerRunner
  46         executor                 *stubExecutor
  47         keepmount                string
  48         keepmountTmp             []string
  49         testDispatcherKeepClient KeepTestClient
  50         testContainerKeepClient  KeepTestClient
  51 }
  52
  53 func (s *TestSuite) SetUpTest(c *C) {
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 133 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 134 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 135 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 136 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 137 func (e *stubExecutor) Close()                          { e.closed = true }
 138 func (e *stubExecutor) Wait(context.Context) (int, error) {
 139         return <-e.exit, e.waitErr
 140 }
 141 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 142         return nil, errors.New("unimplemented")
 143 }
 144 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 145
 146 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 147
 148 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 149 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 150 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 151
 152 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 153 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 154
 155 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 156 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 157 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 158 `
 159
 160 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 161
 162 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 163 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 164
 165 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 166 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 167
 168 func (client *ArvTestClient) Create(resourceType string,
 169         parameters arvadosclient.Dict,
 170         output interface{}) error {
 171
 172         client.Mutex.Lock()
 173         defer client.Mutex.Unlock()
 174
 175         client.Calls++
 176         client.Content = append(client.Content, parameters)
 177
 178         if resourceType == "logs" {
 179                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 180                 if client.Logs == nil {
 181                         client.Logs = make(map[string]*bytes.Buffer)
 182                 }
 183                 if client.Logs[et] == nil {
 184                         client.Logs[et] = &bytes.Buffer{}
 185                 }
 186                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 187         }
 188
 189         if resourceType == "collections" && output != nil {
 190                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 191                 md5sum := md5.Sum([]byte(mt))
 192                 outmap := output.(*arvados.Collection)
 193                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 194                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 195         }
 196
 197         return nil
 198 }
 199
 200 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 201         switch {
 202         case method == "GET" && resourceType == "containers" && action == "auth":
 203                 return json.Unmarshal([]byte(`{
 204                         "kind": "arvados#api_client_authorization",
 205                         "uuid": "`+fakeAuthUUID+`",
 206                         "api_token": "`+fakeAuthToken+`"
 207                         }`), output)
 208         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 209                 if client.secretMounts != nil {
 210                         return json.Unmarshal(client.secretMounts, output)
 211                 }
 212                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 213         default:
 214                 return fmt.Errorf("Not found")
 215         }
 216 }
 217
 218 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 219         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 220         var j []byte
 221         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 222                 j = []byte(`{
 223                         "kind": "arvados#nodeList",
 224                         "items": [{
 225                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 226                                 "hostname": "compute2",
 227                                 "properties": {"total_cpu_cores": 16}
 228                         }]}`)
 229         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 230                 if uuid == "" {
 231                         j, err = json.Marshal(map[string]interface{}{
 232                                 "items": []interface{}{client.Container},
 233                                 "kind":  "arvados#nodeList",
 234                         })
 235                 } else {
 236                         j, err = json.Marshal(client.Container)
 237                 }
 238         } else {
 239                 j = []byte(`{
 240                         "command": ["sleep", "1"],
 241                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 242                         "cwd": ".",
 243                         "environment": {},
 244                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 245                         "output_path": "/tmp",
 246                         "priority": 1,
 247                         "runtime_constraints": {}
 248                 }`)
 249         }
 250         return ioutil.NopCloser(bytes.NewReader(j)), err
 251 }
 252
 253 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 254         if resourceType == "collections" {
 255                 if uuid == hwPDH {
 256                         output.(*arvados.Collection).ManifestText = hwManifest
 257                 } else if uuid == otherPDH {
 258                         output.(*arvados.Collection).ManifestText = otherManifest
 259                 } else if uuid == normalizedWithSubdirsPDH {
 260                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 261                 } else if uuid == denormalizedWithSubdirsPDH {
 262                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 263                 }
 264         }
 265         if resourceType == "containers" {
 266                 (*output.(*arvados.Container)) = client.Container
 267         }
 268         return nil
 269 }
 270
 271 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 272         client.Mutex.Lock()
 273         defer client.Mutex.Unlock()
 274         client.Calls++
 275         client.Content = append(client.Content, parameters)
 276         if resourceType == "containers" {
 277                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 278                         client.WasSetRunning = true
 279                 }
 280         } else if resourceType == "collections" && output != nil {
 281                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 282                 output.(*arvados.Collection).UUID = uuid
 283                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 284         }
 285         return nil
 286 }
 287
 288 var discoveryMap = map[string]interface{}{
 289         "defaultTrashLifetime":               float64(1209600),
 290         "crunchLimitLogBytesPerJob":          float64(67108864),
 291         "crunchLogThrottleBytes":             float64(65536),
 292         "crunchLogThrottlePeriod":            float64(60),
 293         "crunchLogThrottleLines":             float64(1024),
 294         "crunchLogPartialLineThrottlePeriod": float64(5),
 295         "crunchLogBytesPerEvent":             float64(4096),
 296         "crunchLogSecondsBetweenEvents":      float64(1),
 297 }
 298
 299 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 300         return discoveryMap[key], nil
 301 }
 302
 303 // CalledWith returns the parameters from the first API call whose
 304 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 305 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 306 // no call matches, it returns nil.
 307 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 308 call:
 309         for _, content := range client.Content {
 310                 var v interface{} = content
 311                 for _, k := range strings.Split(jpath, ".") {
 312                         if dict, ok := v.(arvadosclient.Dict); !ok {
 313                                 continue call
 314                         } else {
 315                                 v = dict[k]
 316                         }
 317                 }
 318                 if v == expect {
 319                         return content
 320                 }
 321         }
 322         return nil
 323 }
 324
 325 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 326         return locator, nil
 327 }
 328
 329 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 330         client.Content = opts.Data
 331         return arvados.BlockWriteResponse{
 332                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 333         }, nil
 334 }
 335
 336 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 337         return 0, errors.New("not implemented")
 338 }
 339
 340 func (client *KeepTestClient) ClearBlockCache() {
 341 }
 342
 343 func (client *KeepTestClient) Close() {
 344         client.Content = nil
 345 }
 346
 347 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 348         client.StorageClasses = sc
 349 }
 350
 351 type FileWrapper struct {
 352         io.ReadCloser
 353         len int64
 354 }
 355
 356 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 357         return nil, errors.New("not implemented")
 358 }
 359
 360 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 361         return 0, errors.New("not implemented")
 362 }
 363
 364 func (fw FileWrapper) Size() int64 {
 365         return fw.len
 366 }
 367
 368 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 369         return nil, errors.New("not implemented")
 370 }
 371
 372 func (fw FileWrapper) Truncate(int64) error {
 373         return errors.New("not implemented")
 374 }
 375
 376 func (fw FileWrapper) Write([]byte) (int, error) {
 377         return 0, errors.New("not implemented")
 378 }
 379
 380 func (fw FileWrapper) Sync() error {
 381         return errors.New("not implemented")
 382 }
 383
 384 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 385         return nil, errors.New("not implemented")
 386 }
 387
 388 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 389         return errors.New("not implemented")
 390 }
 391
 392 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 393         if filename == hwImageID+".tar" {
 394                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 395                 client.Called = true
 396                 return FileWrapper{rdr, 1321984}, nil
 397         } else if filename == "/file1_in_main.txt" {
 398                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 399                 client.Called = true
 400                 return FileWrapper{rdr, 3}, nil
 401         }
 402         return nil, nil
 403 }
 404
 405 func (s *TestSuite) TestLoadImage(c *C) {
 406         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 407         s.runner.Container.Mounts = map[string]arvados.Mount{
 408                 "/out": {Kind: "tmp", Writable: true},
 409         }
 410         s.runner.Container.OutputPath = "/out"
 411
 412         _, err := s.runner.SetupMounts()
 413         c.Assert(err, IsNil)
 414
 415         imageID, err := s.runner.LoadImage()
 416         c.Check(err, IsNil)
 417         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 418         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 419
 420         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 421         s.executor.imageLoaded = false
 422         s.executor.loaded = ""
 423         s.executor.loadErr = errors.New("bork")
 424         imageID, err = s.runner.LoadImage()
 425         c.Check(err, ErrorMatches, ".*bork")
 426         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 427
 428         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 429         s.executor.imageLoaded = false
 430         s.executor.loaded = ""
 431         s.executor.loadErr = nil
 432         imageID, err = s.runner.LoadImage()
 433         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 434         c.Check(s.executor.loaded, Equals, "")
 435 }
 436
 437 type ArvErrorTestClient struct{}
 438
 439 func (ArvErrorTestClient) Create(resourceType string,
 440         parameters arvadosclient.Dict,
 441         output interface{}) error {
 442         return nil
 443 }
 444
 445 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 446         if method == "GET" && resourceType == "containers" && action == "auth" {
 447                 return nil
 448         }
 449         return errors.New("ArvError")
 450 }
 451
 452 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 453         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 454         return nil, errors.New("ArvError")
 455 }
 456
 457 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 458         return errors.New("ArvError")
 459 }
 460
 461 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 462         return nil
 463 }
 464
 465 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 466         return discoveryMap[key], nil
 467 }
 468
 469 type KeepErrorTestClient struct {
 470         KeepTestClient
 471 }
 472
 473 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 474         return nil, errors.New("KeepError")
 475 }
 476
 477 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 478         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 479 }
 480
 481 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 482         return "", errors.New("KeepError")
 483 }
 484
 485 type KeepReadErrorTestClient struct {
 486         KeepTestClient
 487 }
 488
 489 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 490         return 0, errors.New("KeepError")
 491 }
 492
 493 type ErrorReader struct {
 494         FileWrapper
 495 }
 496
 497 func (ErrorReader) Read(p []byte) (n int, err error) {
 498         return 0, errors.New("ErrorReader")
 499 }
 500
 501 func (ErrorReader) Seek(int64, int) (int64, error) {
 502         return 0, errors.New("ErrorReader")
 503 }
 504
 505 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 506         return ErrorReader{}, nil
 507 }
 508
 509 type ClosableBuffer struct {
 510         bytes.Buffer
 511 }
 512
 513 func (*ClosableBuffer) Close() error {
 514         return nil
 515 }
 516
 517 type TestLogs struct {
 518         Stdout ClosableBuffer
 519         Stderr ClosableBuffer
 520 }
 521
 522 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 523         if logstr == "stdout" {
 524                 return &tl.Stdout, nil
 525         }
 526         if logstr == "stderr" {
 527                 return &tl.Stderr, nil
 528         }
 529         return nil, errors.New("???")
 530 }
 531
 532 func dockerLog(fd byte, msg string) []byte {
 533         by := []byte(msg)
 534         header := make([]byte, 8+len(by))
 535         header[0] = fd
 536         header[7] = byte(len(by))
 537         copy(header[8:], by)
 538         return header
 539 }
 540
 541 func (s *TestSuite) TestRunContainer(c *C) {
 542         s.executor.runFunc = func() {
 543                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 544                 s.executor.exit <- 0
 545         }
 546
 547         var logs TestLogs
 548         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 549         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 550         s.runner.Container.Command = []string{"./hw"}
 551         s.runner.Container.OutputStorageClasses = []string{"default"}
 552
 553         imageID, err := s.runner.LoadImage()
 554         c.Assert(err, IsNil)
 555
 556         err = s.runner.CreateContainer(imageID, nil)
 557         c.Assert(err, IsNil)
 558
 559         err = s.runner.StartContainer()
 560         c.Assert(err, IsNil)
 561
 562         err = s.runner.WaitFinish()
 563         c.Assert(err, IsNil)
 564
 565         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 566         c.Check(logs.Stderr.String(), Equals, "")
 567 }
 568
 569 func (s *TestSuite) TestCommitLogs(c *C) {
 570         api := &ArvTestClient{}
 571         kc := &KeepTestClient{}
 572         defer kc.Close()
 573         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 574         c.Assert(err, IsNil)
 575         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 576
 577         cr.CrunchLog.Print("Hello world!")
 578         cr.CrunchLog.Print("Goodbye")
 579         cr.finalState = "Complete"
 580
 581         err = cr.CommitLogs()
 582         c.Check(err, IsNil)
 583
 584         c.Check(api.Calls, Equals, 2)
 585         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 586         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 587         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 588         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 589 }
 590
 591 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 592         api := &ArvTestClient{}
 593         kc := &KeepTestClient{}
 594         defer kc.Close()
 595         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 596         c.Assert(err, IsNil)
 597
 598         err = cr.UpdateContainerRunning()
 599         c.Check(err, IsNil)
 600
 601         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 602 }
 603
 604 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 605         api := &ArvTestClient{}
 606         kc := &KeepTestClient{}
 607         defer kc.Close()
 608         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 609         c.Assert(err, IsNil)
 610
 611         cr.LogsPDH = new(string)
 612         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 613
 614         cr.ExitCode = new(int)
 615         *cr.ExitCode = 42
 616         cr.finalState = "Complete"
 617
 618         err = cr.UpdateContainerFinal()
 619         c.Check(err, IsNil)
 620
 621         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 622         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 624 }
 625
 626 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 627         api := &ArvTestClient{}
 628         kc := &KeepTestClient{}
 629         defer kc.Close()
 630         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 631         c.Assert(err, IsNil)
 632         cr.cCancelled = true
 633         cr.finalState = "Cancelled"
 634
 635         err = cr.UpdateContainerFinal()
 636         c.Check(err, IsNil)
 637
 638         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 639         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 640         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 641 }
 642
 643 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 644 // dress rehearsal of the Run() function, starting from a JSON container record.
 645 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 646         err := json.Unmarshal([]byte(record), &s.api.Container)
 647         c.Assert(err, IsNil)
 648         initialState := s.api.Container.State
 649
 650         var sm struct {
 651                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 652         }
 653         err = json.Unmarshal([]byte(record), &sm)
 654         c.Check(err, IsNil)
 655         secretMounts, err := json.Marshal(sm)
 656         c.Assert(err, IsNil)
 657         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 658
 659         s.executor.runFunc = func() {
 660                 fn()
 661                 s.executor.exit <- exitCode
 662         }
 663
 664         s.runner.statInterval = 100 * time.Millisecond
 665         s.runner.containerWatchdogInterval = time.Second
 666
 667         realTemp := c.MkDir()
 668         tempcount := 0
 669         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 670                 tempcount++
 671                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 672                 err := os.Mkdir(d, os.ModePerm)
 673                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 674                         // Test case must have pre-populated the tempdir
 675                         err = nil
 676                 }
 677                 return d, err
 678         }
 679         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 680                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 681         }
 682
 683         if extraMounts != nil && len(extraMounts) > 0 {
 684                 err := s.runner.SetupArvMountPoint("keep")
 685                 c.Check(err, IsNil)
 686
 687                 for _, m := range extraMounts {
 688                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 689                 }
 690         }
 691
 692         err = s.runner.Run()
 693         if s.api.CalledWith("container.state", "Complete") != nil {
 694                 c.Check(err, IsNil)
 695         }
 696         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 697                 c.Check(s.api.WasSetRunning, Equals, true)
 698                 var lastupdate arvadosclient.Dict
 699                 for _, content := range s.api.Content {
 700                         if content["container"] != nil {
 701                                 lastupdate = content["container"].(arvadosclient.Dict)
 702                         }
 703                 }
 704                 if lastupdate["log"] == nil {
 705                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 706                 }
 707         }
 708
 709         if err != nil {
 710                 for k, v := range s.api.Logs {
 711                         c.Log(k)
 712                         c.Log(v.String())
 713                 }
 714         }
 715
 716         return s.api, s.runner, realTemp
 717 }
 718
 719 func (s *TestSuite) TestFullRunHello(c *C) {
 720         s.runner.enableMemoryLimit = true
 721         s.runner.networkMode = "default"
 722         s.fullRunHelper(c, `{
 723     "command": ["echo", "hello world"],
 724     "container_image": "`+arvadostest.DockerImage112PDH+`",
 725     "cwd": ".",
 726     "environment": {"foo":"bar","baz":"waz"},
 727     "mounts": {"/tmp": {"kind": "tmp"} },
 728     "output_path": "/tmp",
 729     "priority": 1,
 730     "runtime_constraints": {"vcpus":1,"ram":1000000},
 731     "state": "Locked",
 732     "output_storage_classes": ["default"]
 733 }`, nil, 0, func() {
 734                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 735                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 736                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 737                 c.Check(s.executor.created.VCPUs, Equals, 1)
 738                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 739                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 740                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 741                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 742                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 743         })
 744
 745         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 746         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 747         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 748         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 749         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 750 }
 751
 752 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 753         var ran bool
 754         s.fullRunHelper(c, `{
 755     "command": ["sleep", "3"],
 756     "container_image": "`+arvadostest.DockerImage112PDH+`",
 757     "cwd": ".",
 758     "environment": {},
 759     "mounts": {"/tmp": {"kind": "tmp"} },
 760     "output_path": "/tmp",
 761     "priority": 1,
 762     "runtime_constraints": {},
 763     "scheduling_parameters":{"max_run_time": 1},
 764     "state": "Running"
 765 }`, nil, 2, func() {
 766                 ran = true
 767         })
 768         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 769         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 770         c.Check(ran, Equals, false)
 771 }
 772
 773 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 774         s.fullRunHelper(c, `{
 775     "command": ["sleep", "3"],
 776     "container_image": "`+arvadostest.DockerImage112PDH+`",
 777     "cwd": ".",
 778     "environment": {},
 779     "mounts": {"/tmp": {"kind": "tmp"} },
 780     "output_path": "/tmp",
 781     "priority": 1,
 782     "runtime_constraints": {},
 783     "scheduling_parameters":{"max_run_time": 1},
 784     "state": "Locked"
 785 }`, nil, 0, func() {
 786                 time.Sleep(3 * time.Second)
 787         })
 788
 789         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 790         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 791 }
 792
 793 func (s *TestSuite) TestContainerWaitFails(c *C) {
 794         s.fullRunHelper(c, `{
 795     "command": ["sleep", "3"],
 796     "container_image": "`+arvadostest.DockerImage112PDH+`",
 797     "cwd": ".",
 798     "mounts": {"/tmp": {"kind": "tmp"} },
 799     "output_path": "/tmp",
 800     "priority": 1,
 801     "state": "Locked"
 802 }`, nil, 0, func() {
 803                 s.executor.waitErr = errors.New("Container is not running")
 804         })
 805
 806         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 807         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 808 }
 809
 810 func (s *TestSuite) TestCrunchstat(c *C) {
 811         s.fullRunHelper(c, `{
 812                 "command": ["sleep", "1"],
 813                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 814                 "cwd": ".",
 815                 "environment": {},
 816                 "mounts": {"/tmp": {"kind": "tmp"} },
 817                 "output_path": "/tmp",
 818                 "priority": 1,
 819                 "runtime_constraints": {},
 820                 "state": "Locked"
 821         }`, nil, 0, func() {
 822                 time.Sleep(time.Second)
 823         })
 824
 825         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 826         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 827
 828         // We didn't actually start a container, so crunchstat didn't
 829         // find accounting files and therefore didn't log any stats.
 830         // It should have logged a "can't find accounting files"
 831         // message after one poll interval, though, so we can confirm
 832         // it's alive:
 833         c.Assert(s.api.Logs["crunchstat"], NotNil)
 834         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 835
 836         // The "files never appeared" log assures us that we called
 837         // (*crunchstat.Reporter)Stop(), and that we set it up with
 838         // the correct container ID "abcde":
 839         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 840 }
 841
 842 func (s *TestSuite) TestNodeInfoLog(c *C) {
 843         os.Setenv("SLURMD_NODENAME", "compute2")
 844         s.fullRunHelper(c, `{
 845                 "command": ["sleep", "1"],
 846                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 847                 "cwd": ".",
 848                 "environment": {},
 849                 "mounts": {"/tmp": {"kind": "tmp"} },
 850                 "output_path": "/tmp",
 851                 "priority": 1,
 852                 "runtime_constraints": {},
 853                 "state": "Locked"
 854         }`, nil, 0,
 855                 func() {
 856                         time.Sleep(time.Second)
 857                 })
 858
 859         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 860         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 861
 862         c.Assert(s.api.Logs["node"], NotNil)
 863         json := s.api.Logs["node"].String()
 864         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 865         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 866         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 867
 868         c.Assert(s.api.Logs["node-info"], NotNil)
 869         json = s.api.Logs["node-info"].String()
 870         c.Check(json, Matches, `(?ms).*Host Information.*`)
 871         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 872         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 873         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 874         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 875 }
 876
 877 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 878         s.fullRunHelper(c, `{
 879                 "command": ["sleep", "1"],
 880                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 881                 "cwd": ".",
 882                 "environment": {},
 883                 "mounts": {"/tmp": {"kind": "tmp"} },
 884                 "output_path": "/tmp",
 885                 "priority": 1,
 886                 "runtime_constraints": {},
 887                 "state": "Locked"
 888         }`, nil, 0,
 889                 func() {
 890                 })
 891
 892         c.Assert(s.api.Logs["crunch-run"], NotNil)
 893         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 894         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 895         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 896         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 897 }
 898
 899 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
 900         var collection_create, container_update arvadosclient.Dict
 901         s.fullRunHelper(c, `{
 902                 "command": ["true"],
 903                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 904                 "cwd": ".",
 905                 "environment": {},
 906                 "mounts": {"/tmp": {"kind": "tmp"} },
 907                 "output_path": "/tmp",
 908                 "priority": 1,
 909                 "runtime_constraints": {},
 910                 "state": "Locked",
 911                 "uuid": "zzzzz-dz642-202301121543210"
 912         }`, nil, 0,
 913                 func() {
 914                         collection_create = s.api.CalledWith("ensure_unique_name", true)
 915                         container_update = s.api.CalledWith("container.state", "Running")
 916                 })
 917
 918         c.Assert(collection_create, NotNil)
 919         log_collection := collection_create["collection"].(arvadosclient.Dict)
 920         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
 921         manifest_text := log_collection["manifest_text"].(string)
 922         // We check that the file size is at least two digits as an easy way to
 923         // check the file isn't empty.
 924         c.Check(manifest_text, Matches, `\. .* \d+:\d{2,}:node-info\.txt .*\n`)
 925         c.Check(manifest_text, Matches, `\. .* \d+:\d{2,}:node\.json .*\n`)
 926
 927         c.Assert(container_update, NotNil)
 928         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Matches, `zzzzz-4zz18-[0-9a-z]{15}`)
 929 }
 930
 931 func (s *TestSuite) TestContainerRecordLog(c *C) {
 932         s.fullRunHelper(c, `{
 933                 "command": ["sleep", "1"],
 934                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 935                 "cwd": ".",
 936                 "environment": {},
 937                 "mounts": {"/tmp": {"kind": "tmp"} },
 938                 "output_path": "/tmp",
 939                 "priority": 1,
 940                 "runtime_constraints": {},
 941                 "state": "Locked"
 942         }`, nil, 0,
 943                 func() {
 944                         time.Sleep(time.Second)
 945                 })
 946
 947         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 948         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 949
 950         c.Assert(s.api.Logs["container"], NotNil)
 951         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 952 }
 953
 954 func (s *TestSuite) TestFullRunStderr(c *C) {
 955         s.fullRunHelper(c, `{
 956     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 957     "container_image": "`+arvadostest.DockerImage112PDH+`",
 958     "cwd": ".",
 959     "environment": {},
 960     "mounts": {"/tmp": {"kind": "tmp"} },
 961     "output_path": "/tmp",
 962     "priority": 1,
 963     "runtime_constraints": {},
 964     "state": "Locked"
 965 }`, nil, 1, func() {
 966                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 967                 fmt.Fprintln(s.executor.created.Stderr, "world")
 968         })
 969
 970         final := s.api.CalledWith("container.state", "Complete")
 971         c.Assert(final, NotNil)
 972         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 973         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 974
 975         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 976         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 977 }
 978
 979 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 980         s.fullRunHelper(c, `{
 981     "command": ["pwd"],
 982     "container_image": "`+arvadostest.DockerImage112PDH+`",
 983     "cwd": ".",
 984     "environment": {},
 985     "mounts": {"/tmp": {"kind": "tmp"} },
 986     "output_path": "/tmp",
 987     "priority": 1,
 988     "runtime_constraints": {},
 989     "state": "Locked"
 990 }`, nil, 0, func() {
 991                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 992         })
 993
 994         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 995         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 996         c.Log(s.api.Logs["stdout"])
 997         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 998 }
 999
1000 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1001         s.fullRunHelper(c, `{
1002     "command": ["pwd"],
1003     "container_image": "`+arvadostest.DockerImage112PDH+`",
1004     "cwd": "/bin",
1005     "environment": {},
1006     "mounts": {"/tmp": {"kind": "tmp"} },
1007     "output_path": "/tmp",
1008     "priority": 1,
1009     "runtime_constraints": {},
1010     "state": "Locked"
1011 }`, nil, 0, func() {
1012                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1013         })
1014
1015         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1016         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1017         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1018 }
1019
1020 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1021         s.fullRunHelper(c, `{
1022     "command": ["pwd"],
1023     "container_image": "`+arvadostest.DockerImage112PDH+`",
1024     "cwd": "/bin",
1025     "environment": {},
1026     "mounts": {"/tmp": {"kind": "tmp"} },
1027     "output_path": "/tmp",
1028     "priority": 1,
1029     "runtime_constraints": {},
1030     "state": "Locked",
1031     "output_storage_classes": ["foo", "bar"]
1032 }`, nil, 0, func() {
1033                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1034         })
1035
1036         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1037         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1038         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1039         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1040         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1041 }
1042
1043 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1044         s.fullRunHelper(c, `{
1045     "command": ["pwd"],
1046     "container_image": "`+arvadostest.DockerImage112PDH+`",
1047     "cwd": "/bin",
1048     "environment": {},
1049     "mounts": {"/tmp": {"kind": "tmp"} },
1050     "output_path": "/tmp",
1051     "priority": 1,
1052     "runtime_constraints": {"cuda": {"device_count": 2}},
1053     "state": "Locked",
1054     "output_storage_classes": ["foo", "bar"]
1055 }`, nil, 0, func() {
1056                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1057         })
1058         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1059 }
1060
1061 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1062         s.fullRunHelper(c, `{
1063     "command": ["pwd"],
1064     "container_image": "`+arvadostest.DockerImage112PDH+`",
1065     "cwd": "/bin",
1066     "environment": {},
1067     "mounts": {"/tmp": {"kind": "tmp"} },
1068     "output_path": "/tmp",
1069     "priority": 1,
1070     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1071     "state": "Locked",
1072     "output_storage_classes": ["foo", "bar"]
1073 }`, nil, 0, func() {
1074                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1075         })
1076         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1077 }
1078
1079 func (s *TestSuite) TestStopOnSignal(c *C) {
1080         s.executor.runFunc = func() {
1081                 s.executor.created.Stdout.Write([]byte("foo\n"))
1082                 s.runner.SigChan <- syscall.SIGINT
1083         }
1084         s.testStopContainer(c)
1085 }
1086
1087 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1088         s.executor.runFunc = func() {
1089                 s.executor.created.Stdout.Write([]byte("foo\n"))
1090                 s.runner.ArvMountExit <- nil
1091                 close(s.runner.ArvMountExit)
1092         }
1093         s.runner.ArvMountExit = make(chan error)
1094         s.testStopContainer(c)
1095 }
1096
1097 func (s *TestSuite) testStopContainer(c *C) {
1098         record := `{
1099     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1100     "container_image": "` + arvadostest.DockerImage112PDH + `",
1101     "cwd": ".",
1102     "environment": {},
1103     "mounts": {"/tmp": {"kind": "tmp"} },
1104     "output_path": "/tmp",
1105     "priority": 1,
1106     "runtime_constraints": {},
1107     "state": "Locked"
1108 }`
1109
1110         err := json.Unmarshal([]byte(record), &s.api.Container)
1111         c.Assert(err, IsNil)
1112
1113         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1114         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1115                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1116         }
1117
1118         done := make(chan error)
1119         go func() {
1120                 done <- s.runner.Run()
1121         }()
1122         select {
1123         case <-time.After(20 * time.Second):
1124                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1125                 c.Fatal("timed out")
1126         case err = <-done:
1127                 c.Check(err, IsNil)
1128         }
1129         for k, v := range s.api.Logs {
1130                 c.Log(k)
1131                 c.Log(v.String(), "\n")
1132         }
1133
1134         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1135         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1136         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1137 }
1138
1139 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1140         s.fullRunHelper(c, `{
1141     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1142     "container_image": "`+arvadostest.DockerImage112PDH+`",
1143     "cwd": "/bin",
1144     "environment": {"FROBIZ": "bilbo"},
1145     "mounts": {"/tmp": {"kind": "tmp"} },
1146     "output_path": "/tmp",
1147     "priority": 1,
1148     "runtime_constraints": {},
1149     "state": "Locked"
1150 }`, nil, 0, func() {
1151                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1152         })
1153
1154         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1155         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1156         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1157 }
1158
1159 type ArvMountCmdLine struct {
1160         Cmd   []string
1161         token string
1162 }
1163
1164 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1165         am.Cmd = c
1166         am.token = token
1167         return nil, nil
1168 }
1169
1170 func stubCert(temp string) string {
1171         path := temp + "/ca-certificates.crt"
1172         crt, _ := os.Create(path)
1173         crt.Close()
1174         arvadosclient.CertFiles = []string{path}
1175         return path
1176 }
1177
1178 func (s *TestSuite) TestSetupMounts(c *C) {
1179         cr := s.runner
1180         am := &ArvMountCmdLine{}
1181         cr.RunArvMount = am.ArvMountTest
1182         cr.ContainerArvClient = &ArvTestClient{}
1183         cr.ContainerKeepClient = &KeepTestClient{}
1184         cr.Container.OutputStorageClasses = []string{"default"}
1185
1186         realTemp := c.MkDir()
1187         certTemp := c.MkDir()
1188         stubCertPath := stubCert(certTemp)
1189         cr.parentTemp = realTemp
1190
1191         i := 0
1192         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1193                 i++
1194                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1195                 err := os.Mkdir(d, os.ModePerm)
1196                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1197                         // Test case must have pre-populated the tempdir
1198                         err = nil
1199                 }
1200                 return d, err
1201         }
1202
1203         checkEmpty := func() {
1204                 // Should be deleted.
1205                 _, err := os.Stat(realTemp)
1206                 c.Assert(os.IsNotExist(err), Equals, true)
1207
1208                 // Now recreate it for the next test.
1209                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1210         }
1211
1212         {
1213                 i = 0
1214                 cr.ArvMountPoint = ""
1215                 cr.Container.Mounts = make(map[string]arvados.Mount)
1216                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1217                 cr.Container.OutputPath = "/tmp"
1218                 cr.statInterval = 5 * time.Second
1219                 bindmounts, err := cr.SetupMounts()
1220                 c.Check(err, IsNil)
1221                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1222                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1223                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1224                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1225                 os.RemoveAll(cr.ArvMountPoint)
1226                 cr.CleanupDirs()
1227                 checkEmpty()
1228         }
1229
1230         {
1231                 i = 0
1232                 cr.ArvMountPoint = ""
1233                 cr.Container.Mounts = make(map[string]arvados.Mount)
1234                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1235                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1236                 cr.Container.OutputPath = "/out"
1237                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1238
1239                 bindmounts, err := cr.SetupMounts()
1240                 c.Check(err, IsNil)
1241                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1242                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1243                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1244                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1245                 os.RemoveAll(cr.ArvMountPoint)
1246                 cr.CleanupDirs()
1247                 checkEmpty()
1248         }
1249
1250         {
1251                 i = 0
1252                 cr.ArvMountPoint = ""
1253                 cr.Container.Mounts = make(map[string]arvados.Mount)
1254                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1255                 cr.Container.OutputPath = "/tmp"
1256                 cr.Container.RuntimeConstraints.API = true
1257                 cr.Container.OutputStorageClasses = []string{"default"}
1258
1259                 bindmounts, err := cr.SetupMounts()
1260                 c.Check(err, IsNil)
1261                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1262                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1263                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1264                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1265                 os.RemoveAll(cr.ArvMountPoint)
1266                 cr.CleanupDirs()
1267                 checkEmpty()
1268
1269                 cr.Container.RuntimeConstraints.API = false
1270         }
1271
1272         {
1273                 i = 0
1274                 cr.ArvMountPoint = ""
1275                 cr.Container.Mounts = map[string]arvados.Mount{
1276                         "/keeptmp": {Kind: "collection", Writable: true},
1277                 }
1278                 cr.Container.OutputPath = "/keeptmp"
1279
1280                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1281
1282                 bindmounts, err := cr.SetupMounts()
1283                 c.Check(err, IsNil)
1284                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1285                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1286                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1287                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1288                 os.RemoveAll(cr.ArvMountPoint)
1289                 cr.CleanupDirs()
1290                 checkEmpty()
1291         }
1292
1293         {
1294                 i = 0
1295                 cr.ArvMountPoint = ""
1296                 cr.Container.Mounts = map[string]arvados.Mount{
1297                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1298                         "/keepout": {Kind: "collection", Writable: true},
1299                 }
1300                 cr.Container.OutputPath = "/keepout"
1301
1302                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1303                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1304
1305                 bindmounts, err := cr.SetupMounts()
1306                 c.Check(err, IsNil)
1307                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1308                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1309                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1310                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1311                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1312                         "/keepout": {realTemp + "/keep1/tmp0", false},
1313                 })
1314                 os.RemoveAll(cr.ArvMountPoint)
1315                 cr.CleanupDirs()
1316                 checkEmpty()
1317         }
1318
1319         {
1320                 i = 0
1321                 cr.ArvMountPoint = ""
1322                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1323                 cr.Container.Mounts = map[string]arvados.Mount{
1324                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1325                         "/keepout": {Kind: "collection", Writable: true},
1326                 }
1327                 cr.Container.OutputPath = "/keepout"
1328
1329                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1330                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1331
1332                 bindmounts, err := cr.SetupMounts()
1333                 c.Check(err, IsNil)
1334                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1335                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1336                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1337                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1338                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1339                         "/keepout": {realTemp + "/keep1/tmp0", false},
1340                 })
1341                 os.RemoveAll(cr.ArvMountPoint)
1342                 cr.CleanupDirs()
1343                 checkEmpty()
1344         }
1345
1346         for _, test := range []struct {
1347                 in  interface{}
1348                 out string
1349         }{
1350                 {in: "foo", out: `"foo"`},
1351                 {in: nil, out: `null`},
1352                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1353         } {
1354                 i = 0
1355                 cr.ArvMountPoint = ""
1356                 cr.Container.Mounts = map[string]arvados.Mount{
1357                         "/mnt/test.json": {Kind: "json", Content: test.in},
1358                 }
1359                 bindmounts, err := cr.SetupMounts()
1360                 c.Check(err, IsNil)
1361                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1362                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1363                 })
1364                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1365                 c.Check(err, IsNil)
1366                 c.Check(content, DeepEquals, []byte(test.out))
1367                 os.RemoveAll(cr.ArvMountPoint)
1368                 cr.CleanupDirs()
1369                 checkEmpty()
1370         }
1371
1372         for _, test := range []struct {
1373                 in  interface{}
1374                 out string
1375         }{
1376                 {in: "foo", out: `foo`},
1377                 {in: nil, out: "error"},
1378                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1379         } {
1380                 i = 0
1381                 cr.ArvMountPoint = ""
1382                 cr.Container.Mounts = map[string]arvados.Mount{
1383                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1384                 }
1385                 bindmounts, err := cr.SetupMounts()
1386                 if test.out == "error" {
1387                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1388                 } else {
1389                         c.Check(err, IsNil)
1390                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1391                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1392                         })
1393                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1394                         c.Check(err, IsNil)
1395                         c.Check(content, DeepEquals, []byte(test.out))
1396                 }
1397                 os.RemoveAll(cr.ArvMountPoint)
1398                 cr.CleanupDirs()
1399                 checkEmpty()
1400         }
1401
1402         // Read-only mount points are allowed underneath output_dir mount point
1403         {
1404                 i = 0
1405                 cr.ArvMountPoint = ""
1406                 cr.Container.Mounts = make(map[string]arvados.Mount)
1407                 cr.Container.Mounts = map[string]arvados.Mount{
1408                         "/tmp":     {Kind: "tmp"},
1409                         "/tmp/foo": {Kind: "collection"},
1410                 }
1411                 cr.Container.OutputPath = "/tmp"
1412
1413                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1414
1415                 bindmounts, err := cr.SetupMounts()
1416                 c.Check(err, IsNil)
1417                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1418                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1419                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1420                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1421                         "/tmp":     {realTemp + "/tmp2", false},
1422                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1423                 })
1424                 os.RemoveAll(cr.ArvMountPoint)
1425                 cr.CleanupDirs()
1426                 checkEmpty()
1427         }
1428
1429         // Writable mount points copied to output_dir mount point
1430         {
1431                 i = 0
1432                 cr.ArvMountPoint = ""
1433                 cr.Container.Mounts = make(map[string]arvados.Mount)
1434                 cr.Container.Mounts = map[string]arvados.Mount{
1435                         "/tmp": {Kind: "tmp"},
1436                         "/tmp/foo": {Kind: "collection",
1437                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1438                                 Writable:         true},
1439                         "/tmp/bar": {Kind: "collection",
1440                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1441                                 Path:             "baz",
1442                                 Writable:         true},
1443                 }
1444                 cr.Container.OutputPath = "/tmp"
1445
1446                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1447                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1448
1449                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1450                 rf.Write([]byte("bar"))
1451                 rf.Close()
1452
1453                 _, err := cr.SetupMounts()
1454                 c.Check(err, IsNil)
1455                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1456                 c.Check(err, IsNil)
1457                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1458                 c.Check(err, IsNil)
1459                 os.RemoveAll(cr.ArvMountPoint)
1460                 cr.CleanupDirs()
1461                 checkEmpty()
1462         }
1463
1464         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1465         {
1466                 i = 0
1467                 cr.ArvMountPoint = ""
1468                 cr.Container.Mounts = make(map[string]arvados.Mount)
1469                 cr.Container.Mounts = map[string]arvados.Mount{
1470                         "/tmp":     {Kind: "tmp"},
1471                         "/tmp/foo": {Kind: "tmp"},
1472                 }
1473                 cr.Container.OutputPath = "/tmp"
1474
1475                 _, err := cr.SetupMounts()
1476                 c.Check(err, NotNil)
1477                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1478                 os.RemoveAll(cr.ArvMountPoint)
1479                 cr.CleanupDirs()
1480                 checkEmpty()
1481         }
1482
1483         // Only mount point of kind 'collection' is allowed for stdin
1484         {
1485                 i = 0
1486                 cr.ArvMountPoint = ""
1487                 cr.Container.Mounts = make(map[string]arvados.Mount)
1488                 cr.Container.Mounts = map[string]arvados.Mount{
1489                         "stdin": {Kind: "tmp"},
1490                 }
1491
1492                 _, err := cr.SetupMounts()
1493                 c.Check(err, NotNil)
1494                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1495                 os.RemoveAll(cr.ArvMountPoint)
1496                 cr.CleanupDirs()
1497                 checkEmpty()
1498         }
1499
1500         // git_tree mounts
1501         {
1502                 i = 0
1503                 cr.ArvMountPoint = ""
1504                 (*GitMountSuite)(nil).useTestGitServer(c)
1505                 cr.token = arvadostest.ActiveToken
1506                 cr.Container.Mounts = make(map[string]arvados.Mount)
1507                 cr.Container.Mounts = map[string]arvados.Mount{
1508                         "/tip": {
1509                                 Kind:   "git_tree",
1510                                 UUID:   arvadostest.Repository2UUID,
1511                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1512                                 Path:   "/",
1513                         },
1514                         "/non-tip": {
1515                                 Kind:   "git_tree",
1516                                 UUID:   arvadostest.Repository2UUID,
1517                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1518                                 Path:   "/",
1519                         },
1520                 }
1521                 cr.Container.OutputPath = "/tmp"
1522
1523                 bindmounts, err := cr.SetupMounts()
1524                 c.Check(err, IsNil)
1525
1526                 for path, mount := range bindmounts {
1527                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1528                 }
1529
1530                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1531                 c.Check(err, IsNil)
1532                 c.Check(string(data), Equals, "\000\001\002\003")
1533                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1534                 c.Check(err, FitsTypeOf, &os.PathError{})
1535                 c.Check(os.IsNotExist(err), Equals, true)
1536
1537                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1538                 c.Check(err, IsNil)
1539                 c.Check(string(data), Equals, "\000\001\002\003")
1540                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1541                 c.Check(err, IsNil)
1542                 c.Check(string(data), Equals, "testfile\n")
1543
1544                 cr.CleanupDirs()
1545                 checkEmpty()
1546         }
1547 }
1548
1549 func (s *TestSuite) TestStdout(c *C) {
1550         helperRecord := `{
1551                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1552                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1553                 "cwd": "/bin",
1554                 "environment": {"FROBIZ": "bilbo"},
1555                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1556                 "output_path": "/tmp",
1557                 "priority": 1,
1558                 "runtime_constraints": {},
1559                 "state": "Locked"
1560         }`
1561
1562         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1563                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1564         })
1565
1566         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1567         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1568         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1569 }
1570
1571 // Used by the TestStdoutWithWrongPath*()
1572 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1573         err := json.Unmarshal([]byte(record), &s.api.Container)
1574         c.Assert(err, IsNil)
1575         s.executor.runFunc = fn
1576         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1577         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1578                 return s.api, &KeepTestClient{}, nil, nil
1579         }
1580         return s.api, s.runner, s.runner.Run()
1581 }
1582
1583 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1584         _, _, err := s.stdoutErrorRunHelper(c, `{
1585     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1586     "output_path": "/tmp",
1587     "state": "Locked"
1588 }`, func() {})
1589         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1590 }
1591
1592 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1593         _, _, err := s.stdoutErrorRunHelper(c, `{
1594     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1595     "output_path": "/tmp",
1596     "state": "Locked"
1597 }`, func() {})
1598         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1599 }
1600
1601 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1602         _, _, err := s.stdoutErrorRunHelper(c, `{
1603     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1604     "output_path": "/tmp",
1605     "state": "Locked"
1606 }`, func() {})
1607         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1608 }
1609
1610 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1611         s.fullRunHelper(c, `{
1612     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1613     "container_image": "`+arvadostest.DockerImage112PDH+`",
1614     "cwd": "/bin",
1615     "environment": {},
1616     "mounts": {"/tmp": {"kind": "tmp"} },
1617     "output_path": "/tmp",
1618     "priority": 1,
1619     "runtime_constraints": {"API": true},
1620     "state": "Locked"
1621 }`, nil, 0, func() {
1622                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1623                 s.executor.exit <- 3
1624         })
1625         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1626         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1627         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1628 }
1629
1630 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1631         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1632         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1633         s.fullRunHelper(c, `{
1634     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1635     "container_image": "`+arvadostest.DockerImage112PDH+`",
1636     "cwd": "/bin",
1637     "environment": {},
1638     "mounts": {"/tmp": {"kind": "tmp"} },
1639     "output_path": "/tmp",
1640     "priority": 1,
1641     "runtime_constraints": {"API": true},
1642     "state": "Locked"
1643 }`, nil, 0, func() {
1644                 s.api.Container.Output = arvadostest.DockerImage112PDH
1645         })
1646
1647         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1648         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1649         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1650 }
1651
1652 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1653         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1654                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1655                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1656                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1657         }
1658         s.executor.runFunc = func() {
1659                 time.Sleep(time.Second)
1660                 s.executor.exit <- 137
1661         }
1662         record := `{
1663     "command": ["sleep", "1"],
1664     "container_image": "` + arvadostest.DockerImage112PDH + `",
1665     "cwd": "/bin",
1666     "environment": {},
1667     "mounts": {"/tmp": {"kind": "tmp"} },
1668     "output_path": "/tmp",
1669     "priority": 1,
1670     "runtime_constraints": {"API": true},
1671     "state": "Locked"
1672 }`
1673         err := json.Unmarshal([]byte(record), &s.api.Container)
1674         c.Assert(err, IsNil)
1675         err = s.runner.Run()
1676         c.Assert(err, IsNil)
1677         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1678         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1679         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1680         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1681         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1682 }
1683
1684 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1685         helperRecord := `{
1686                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1687                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1688                 "cwd": "/bin",
1689                 "environment": {"FROBIZ": "bilbo"},
1690                 "mounts": {
1691         "/tmp": {"kind": "tmp"},
1692         "/tmp/foo": {"kind": "collection",
1693                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1694                      "exclude_from_output": true
1695         },
1696         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1697     },
1698                 "output_path": "/tmp",
1699                 "priority": 1,
1700                 "runtime_constraints": {},
1701                 "state": "Locked"
1702         }`
1703
1704         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1705
1706         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1707                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1708         })
1709
1710         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1711         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1712         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1713 }
1714
1715 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1716         helperRecord := `{
1717                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1718                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1719                 "cwd": "/bin",
1720                 "environment": {"FROBIZ": "bilbo"},
1721                 "mounts": {
1722         "/tmp": {"kind": "tmp"},
1723         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1724         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1725         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1726         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1727         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1728     },
1729                 "output_path": "/tmp",
1730                 "priority": 1,
1731                 "runtime_constraints": {},
1732                 "state": "Locked",
1733                 "uuid": "zzzzz-dz642-202301130848001"
1734         }`
1735
1736         extraMounts := []string{
1737                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1738                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1739                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1740         }
1741
1742         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1743                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1744         })
1745
1746         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1747                 "/tmp":                   {realtemp + "/tmp1", false},
1748                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1749                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1750                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1751                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1752         })
1753
1754         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1755         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1756         output_count := uint(0)
1757         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1758                 if v["collection"] == nil {
1759                         continue
1760                 }
1761                 collection := v["collection"].(arvadosclient.Dict)
1762                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
1763                         continue
1764                 }
1765                 c.Check(v["ensure_unique_name"], Equals, true)
1766                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1767 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1768 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1769 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1770 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1771 `)
1772                 output_count++
1773         }
1774         c.Check(output_count, Not(Equals), uint(0))
1775 }
1776
1777 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1778         helperRecord := `{
1779                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1780                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1781                 "cwd": "/bin",
1782                 "environment": {"FROBIZ": "bilbo"},
1783                 "mounts": {
1784         "/tmp": {"kind": "tmp"},
1785         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1786         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1787     },
1788                 "output_path": "/tmp",
1789                 "priority": 1,
1790                 "runtime_constraints": {},
1791                 "state": "Locked",
1792                 "uuid": "zzzzz-dz642-202301130848002"
1793         }`
1794
1795         extraMounts := []string{
1796                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1797         }
1798
1799         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1800                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1801         })
1802
1803         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1804         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1805         output_count := uint(0)
1806         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1807                 if v["collection"] == nil {
1808                         continue
1809                 }
1810                 collection := v["collection"].(arvadosclient.Dict)
1811                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
1812                         continue
1813                 }
1814                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1815 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1816 `)
1817                 output_count++
1818         }
1819         c.Check(output_count, Not(Equals), uint(0))
1820 }
1821
1822 func (s *TestSuite) TestOutputError(c *C) {
1823         helperRecord := `{
1824                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1825                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1826                 "cwd": "/bin",
1827                 "environment": {"FROBIZ": "bilbo"},
1828                 "mounts": {
1829                         "/tmp": {"kind": "tmp"}
1830                 },
1831                 "output_path": "/tmp",
1832                 "priority": 1,
1833                 "runtime_constraints": {},
1834                 "state": "Locked"
1835         }`
1836         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1837                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1838         })
1839
1840         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1841 }
1842
1843 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1844         helperRecord := `{
1845                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1846                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1847                 "cwd": "/bin",
1848                 "environment": {"FROBIZ": "bilbo"},
1849                 "mounts": {
1850         "/tmp": {"kind": "tmp"},
1851         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1852         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1853     },
1854                 "output_path": "/tmp",
1855                 "priority": 1,
1856                 "runtime_constraints": {},
1857                 "state": "Locked"
1858         }`
1859
1860         extraMounts := []string{
1861                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1862         }
1863
1864         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1865                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1866         })
1867
1868         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1869         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1870         for _, v := range api.Content {
1871                 if v["collection"] != nil {
1872                         collection := v["collection"].(arvadosclient.Dict)
1873                         if strings.Index(collection["name"].(string), "output") == 0 {
1874                                 manifest := collection["manifest_text"].(string)
1875                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1876 `)
1877                         }
1878                 }
1879         }
1880 }
1881
1882 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1883         helperRecord := `{
1884                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1885                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1886                 "cwd": "/bin",
1887                 "environment": {"FROBIZ": "bilbo"},
1888                 "mounts": {
1889         "/tmp": {"kind": "tmp"},
1890         "stdin": {"kind": "json", "content": "foo"},
1891         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1892     },
1893                 "output_path": "/tmp",
1894                 "priority": 1,
1895                 "runtime_constraints": {},
1896                 "state": "Locked"
1897         }`
1898
1899         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1900                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1901         })
1902
1903         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1904         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1905         for _, v := range api.Content {
1906                 if v["collection"] != nil {
1907                         collection := v["collection"].(arvadosclient.Dict)
1908                         if strings.Index(collection["name"].(string), "output") == 0 {
1909                                 manifest := collection["manifest_text"].(string)
1910                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1911 `)
1912                         }
1913                 }
1914         }
1915 }
1916
1917 func (s *TestSuite) TestStderrMount(c *C) {
1918         api, cr, _ := s.fullRunHelper(c, `{
1919     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1920     "container_image": "`+arvadostest.DockerImage112PDH+`",
1921     "cwd": ".",
1922     "environment": {},
1923     "mounts": {"/tmp": {"kind": "tmp"},
1924                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1925                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1926     "output_path": "/tmp",
1927     "priority": 1,
1928     "runtime_constraints": {},
1929     "state": "Locked"
1930 }`, nil, 1, func() {
1931                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1932                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1933         })
1934
1935         final := api.CalledWith("container.state", "Complete")
1936         c.Assert(final, NotNil)
1937         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1938         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1939
1940         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1941 }
1942
1943 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1944         s.api.callraw = true
1945         err := s.runner.fetchContainerRecord()
1946         c.Assert(err, IsNil)
1947         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1948         c.Logf("%#v", s.runner.Container)
1949         c.Check(err, IsNil)
1950         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1951 }
1952
1953 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1954         nextState := ""
1955         for _, setup := range []func(){
1956                 func() {
1957                         c.Log("// waitErr = ocl runtime error")
1958                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1959                         nextState = "Cancelled"
1960                 },
1961                 func() {
1962                         c.Log("// loadErr = cannot connect")
1963                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1964                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1965                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1966                         c.Assert(err, IsNil)
1967                         nextState = "Queued"
1968                 },
1969         } {
1970                 s.SetUpTest(c)
1971                 setup()
1972                 s.fullRunHelper(c, `{
1973     "command": ["echo", "hello world"],
1974     "container_image": "`+arvadostest.DockerImage112PDH+`",
1975     "cwd": ".",
1976     "environment": {},
1977     "mounts": {"/tmp": {"kind": "tmp"} },
1978     "output_path": "/tmp",
1979     "priority": 1,
1980     "runtime_constraints": {},
1981     "state": "Locked"
1982 }`, nil, 0, func() {})
1983                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1984                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1985                 if s.runner.brokenNodeHook != "" {
1986                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1987                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1988                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1989                 } else {
1990                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1991                 }
1992         }
1993 }
1994
1995 func (s *TestSuite) TestBadCommand(c *C) {
1996         for _, startError := range []string{
1997                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1998                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1999                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2000         } {
2001                 s.SetUpTest(c)
2002                 s.executor.startErr = errors.New(startError)
2003                 s.fullRunHelper(c, `{
2004     "command": ["echo", "hello world"],
2005     "container_image": "`+arvadostest.DockerImage112PDH+`",
2006     "cwd": ".",
2007     "environment": {},
2008     "mounts": {"/tmp": {"kind": "tmp"} },
2009     "output_path": "/tmp",
2010     "priority": 1,
2011     "runtime_constraints": {},
2012     "state": "Locked"
2013 }`, nil, 0, func() {})
2014                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2015                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2016         }
2017 }
2018
2019 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2020         helperRecord := `{
2021                 "command": ["true"],
2022                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2023                 "cwd": "/bin",
2024                 "mounts": {
2025                     "/tmp": {"kind": "tmp"},
2026                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2027                 },
2028                 "secret_mounts": {
2029                 },
2030                 "output_path": "/tmp",
2031                 "priority": 1,
2032                 "runtime_constraints": {},
2033                 "state": "Locked"
2034         }`
2035
2036         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2037                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2038                 c.Check(err, IsNil)
2039                 c.Check(string(content), Equals, "mypassword")
2040         })
2041
2042         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2043         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2044         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2045         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2046
2047         // under secret mounts, not captured in output
2048         helperRecord = `{
2049                 "command": ["true"],
2050                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2051                 "cwd": "/bin",
2052                 "mounts": {
2053                     "/tmp": {"kind": "tmp"}
2054                 },
2055                 "secret_mounts": {
2056                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2057                 },
2058                 "output_path": "/tmp",
2059                 "priority": 1,
2060                 "runtime_constraints": {},
2061                 "state": "Locked"
2062         }`
2063
2064         s.SetUpTest(c)
2065         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2066                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2067                 c.Check(err, IsNil)
2068                 c.Check(string(content), Equals, "mypassword")
2069         })
2070
2071         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2072         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2073         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2074         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2075
2076         // under secret mounts, output dir is a collection, not captured in output
2077         helperRecord = `{
2078                 "command": ["true"],
2079                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2080                 "cwd": "/bin",
2081                 "mounts": {
2082                     "/tmp": {"kind": "collection", "writable": true}
2083                 },
2084                 "secret_mounts": {
2085                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2086                 },
2087                 "output_path": "/tmp",
2088                 "priority": 1,
2089                 "runtime_constraints": {},
2090                 "state": "Locked"
2091         }`
2092
2093         s.SetUpTest(c)
2094         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2095                 // secret.conf should be provisioned as a separate
2096                 // bind mount, i.e., it should not appear in the
2097                 // (fake) fuse filesystem as viewed from the host.
2098                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2099                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2100                         c.Logf("secret.conf: content %q, err %#v", content, err)
2101                 }
2102                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2103                 c.Check(err, IsNil)
2104         })
2105
2106         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2107         c.Check(err, IsNil)
2108         c.Check(string(content), Equals, "mypassword")
2109         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2110         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2111         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2112         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2113 }
2114
2115 type FakeProcess struct {
2116         cmdLine []string
2117 }
2118
2119 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2120         return fp.cmdLine, nil
2121 }