lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/cmd"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/manifest"
  30         "golang.org/x/net/context"
  31
  32         . "gopkg.in/check.v1"
  33 )
  34
  35 // Gocheck boilerplate
  36 func TestCrunchExec(t *testing.T) {
  37         TestingT(t)
  38 }
  39
  40 var _ = Suite(&TestSuite{})
  41
  42 type TestSuite struct {
  43         client                   *arvados.Client
  44         api                      *ArvTestClient
  45         runner                   *ContainerRunner
  46         executor                 *stubExecutor
  47         keepmount                string
  48         keepmountTmp             []string
  49         testDispatcherKeepClient KeepTestClient
  50         testContainerKeepClient  KeepTestClient
  51 }
  52
  53 func (s *TestSuite) SetUpTest(c *C) {
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Version() string                 { return "stub " + cmd.Version.String() }
 133 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 134 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 135 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 136 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 137 func (e *stubExecutor) Close()                          { e.closed = true }
 138 func (e *stubExecutor) Wait(context.Context) (int, error) {
 139         return <-e.exit, e.waitErr
 140 }
 141 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 142         return nil, errors.New("unimplemented")
 143 }
 144 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 145
 146 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 147
 148 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 149 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 150 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 151
 152 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 153 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 154
 155 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 156 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 157 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 158 `
 159
 160 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 161
 162 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 163 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 164
 165 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 166 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 167
 168 func (client *ArvTestClient) Create(resourceType string,
 169         parameters arvadosclient.Dict,
 170         output interface{}) error {
 171
 172         client.Mutex.Lock()
 173         defer client.Mutex.Unlock()
 174
 175         client.Calls++
 176         client.Content = append(client.Content, parameters)
 177
 178         if resourceType == "logs" {
 179                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 180                 if client.Logs == nil {
 181                         client.Logs = make(map[string]*bytes.Buffer)
 182                 }
 183                 if client.Logs[et] == nil {
 184                         client.Logs[et] = &bytes.Buffer{}
 185                 }
 186                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 187         }
 188
 189         if resourceType == "collections" && output != nil {
 190                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 191                 md5sum := md5.Sum([]byte(mt))
 192                 outmap := output.(*arvados.Collection)
 193                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5sum, len(mt))
 194                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%015x", md5sum[:7])
 195         }
 196
 197         return nil
 198 }
 199
 200 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 201         switch {
 202         case method == "GET" && resourceType == "containers" && action == "auth":
 203                 return json.Unmarshal([]byte(`{
 204                         "kind": "arvados#api_client_authorization",
 205                         "uuid": "`+fakeAuthUUID+`",
 206                         "api_token": "`+fakeAuthToken+`"
 207                         }`), output)
 208         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 209                 if client.secretMounts != nil {
 210                         return json.Unmarshal(client.secretMounts, output)
 211                 }
 212                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 213         default:
 214                 return fmt.Errorf("Not found")
 215         }
 216 }
 217
 218 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 219         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 220         var j []byte
 221         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 222                 j = []byte(`{
 223                         "kind": "arvados#nodeList",
 224                         "items": [{
 225                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 226                                 "hostname": "compute2",
 227                                 "properties": {"total_cpu_cores": 16}
 228                         }]}`)
 229         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 230                 if uuid == "" {
 231                         j, err = json.Marshal(map[string]interface{}{
 232                                 "items": []interface{}{client.Container},
 233                                 "kind":  "arvados#nodeList",
 234                         })
 235                 } else {
 236                         j, err = json.Marshal(client.Container)
 237                 }
 238         } else {
 239                 j = []byte(`{
 240                         "command": ["sleep", "1"],
 241                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 242                         "cwd": ".",
 243                         "environment": {},
 244                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 245                         "output_path": "/tmp",
 246                         "priority": 1,
 247                         "runtime_constraints": {}
 248                 }`)
 249         }
 250         return ioutil.NopCloser(bytes.NewReader(j)), err
 251 }
 252
 253 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 254         if resourceType == "collections" {
 255                 if uuid == hwPDH {
 256                         output.(*arvados.Collection).ManifestText = hwManifest
 257                 } else if uuid == otherPDH {
 258                         output.(*arvados.Collection).ManifestText = otherManifest
 259                 } else if uuid == normalizedWithSubdirsPDH {
 260                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 261                 } else if uuid == denormalizedWithSubdirsPDH {
 262                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 263                 }
 264         }
 265         if resourceType == "containers" {
 266                 (*output.(*arvados.Container)) = client.Container
 267         }
 268         return nil
 269 }
 270
 271 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 272         client.Mutex.Lock()
 273         defer client.Mutex.Unlock()
 274         client.Calls++
 275         client.Content = append(client.Content, parameters)
 276         if resourceType == "containers" {
 277                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 278                         client.WasSetRunning = true
 279                 }
 280         } else if resourceType == "collections" && output != nil {
 281                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 282                 output.(*arvados.Collection).UUID = uuid
 283                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 284         }
 285         return nil
 286 }
 287
 288 var discoveryMap = map[string]interface{}{
 289         "defaultTrashLifetime":               float64(1209600),
 290         "crunchLimitLogBytesPerJob":          float64(67108864),
 291         "crunchLogThrottleBytes":             float64(65536),
 292         "crunchLogThrottlePeriod":            float64(60),
 293         "crunchLogThrottleLines":             float64(1024),
 294         "crunchLogPartialLineThrottlePeriod": float64(5),
 295         "crunchLogBytesPerEvent":             float64(4096),
 296         "crunchLogSecondsBetweenEvents":      float64(1),
 297 }
 298
 299 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 300         return discoveryMap[key], nil
 301 }
 302
 303 // CalledWith returns the parameters from the first API call whose
 304 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 305 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 306 // no call matches, it returns nil.
 307 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 308 call:
 309         for _, content := range client.Content {
 310                 var v interface{} = content
 311                 for _, k := range strings.Split(jpath, ".") {
 312                         if dict, ok := v.(arvadosclient.Dict); !ok {
 313                                 continue call
 314                         } else {
 315                                 v = dict[k]
 316                         }
 317                 }
 318                 if v == expect {
 319                         return content
 320                 }
 321         }
 322         return nil
 323 }
 324
 325 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 326         return locator, nil
 327 }
 328
 329 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 330         client.Content = opts.Data
 331         return arvados.BlockWriteResponse{
 332                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 333         }, nil
 334 }
 335
 336 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 337         return 0, errors.New("not implemented")
 338 }
 339
 340 func (client *KeepTestClient) ClearBlockCache() {
 341 }
 342
 343 func (client *KeepTestClient) Close() {
 344         client.Content = nil
 345 }
 346
 347 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 348         client.StorageClasses = sc
 349 }
 350
 351 type FileWrapper struct {
 352         io.ReadCloser
 353         len int64
 354 }
 355
 356 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 357         return nil, errors.New("not implemented")
 358 }
 359
 360 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 361         return 0, errors.New("not implemented")
 362 }
 363
 364 func (fw FileWrapper) Size() int64 {
 365         return fw.len
 366 }
 367
 368 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 369         return nil, errors.New("not implemented")
 370 }
 371
 372 func (fw FileWrapper) Truncate(int64) error {
 373         return errors.New("not implemented")
 374 }
 375
 376 func (fw FileWrapper) Write([]byte) (int, error) {
 377         return 0, errors.New("not implemented")
 378 }
 379
 380 func (fw FileWrapper) Sync() error {
 381         return errors.New("not implemented")
 382 }
 383
 384 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 385         return nil, errors.New("not implemented")
 386 }
 387
 388 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 389         return errors.New("not implemented")
 390 }
 391
 392 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 393         if filename == hwImageID+".tar" {
 394                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 395                 client.Called = true
 396                 return FileWrapper{rdr, 1321984}, nil
 397         } else if filename == "/file1_in_main.txt" {
 398                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 399                 client.Called = true
 400                 return FileWrapper{rdr, 3}, nil
 401         }
 402         return nil, nil
 403 }
 404
 405 func (s *TestSuite) TestLoadImage(c *C) {
 406         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 407         s.runner.Container.Mounts = map[string]arvados.Mount{
 408                 "/out": {Kind: "tmp", Writable: true},
 409         }
 410         s.runner.Container.OutputPath = "/out"
 411
 412         _, err := s.runner.SetupMounts()
 413         c.Assert(err, IsNil)
 414
 415         imageID, err := s.runner.LoadImage()
 416         c.Check(err, IsNil)
 417         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 418         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 419
 420         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 421         s.executor.imageLoaded = false
 422         s.executor.loaded = ""
 423         s.executor.loadErr = errors.New("bork")
 424         imageID, err = s.runner.LoadImage()
 425         c.Check(err, ErrorMatches, ".*bork")
 426         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 427
 428         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 429         s.executor.imageLoaded = false
 430         s.executor.loaded = ""
 431         s.executor.loadErr = nil
 432         imageID, err = s.runner.LoadImage()
 433         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 434         c.Check(s.executor.loaded, Equals, "")
 435 }
 436
 437 type ArvErrorTestClient struct{}
 438
 439 func (ArvErrorTestClient) Create(resourceType string,
 440         parameters arvadosclient.Dict,
 441         output interface{}) error {
 442         return nil
 443 }
 444
 445 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 446         if method == "GET" && resourceType == "containers" && action == "auth" {
 447                 return nil
 448         }
 449         return errors.New("ArvError")
 450 }
 451
 452 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 453         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 454         return nil, errors.New("ArvError")
 455 }
 456
 457 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 458         return errors.New("ArvError")
 459 }
 460
 461 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 462         return nil
 463 }
 464
 465 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 466         return discoveryMap[key], nil
 467 }
 468
 469 type KeepErrorTestClient struct {
 470         KeepTestClient
 471 }
 472
 473 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 474         return nil, errors.New("KeepError")
 475 }
 476
 477 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 478         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 479 }
 480
 481 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 482         return "", errors.New("KeepError")
 483 }
 484
 485 type KeepReadErrorTestClient struct {
 486         KeepTestClient
 487 }
 488
 489 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 490         return 0, errors.New("KeepError")
 491 }
 492
 493 type ErrorReader struct {
 494         FileWrapper
 495 }
 496
 497 func (ErrorReader) Read(p []byte) (n int, err error) {
 498         return 0, errors.New("ErrorReader")
 499 }
 500
 501 func (ErrorReader) Seek(int64, int) (int64, error) {
 502         return 0, errors.New("ErrorReader")
 503 }
 504
 505 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 506         return ErrorReader{}, nil
 507 }
 508
 509 type ClosableBuffer struct {
 510         bytes.Buffer
 511 }
 512
 513 func (*ClosableBuffer) Close() error {
 514         return nil
 515 }
 516
 517 type TestLogs struct {
 518         Stdout ClosableBuffer
 519         Stderr ClosableBuffer
 520 }
 521
 522 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 523         if logstr == "stdout" {
 524                 return &tl.Stdout, nil
 525         }
 526         if logstr == "stderr" {
 527                 return &tl.Stderr, nil
 528         }
 529         return nil, errors.New("???")
 530 }
 531
 532 func dockerLog(fd byte, msg string) []byte {
 533         by := []byte(msg)
 534         header := make([]byte, 8+len(by))
 535         header[0] = fd
 536         header[7] = byte(len(by))
 537         copy(header[8:], by)
 538         return header
 539 }
 540
 541 func (s *TestSuite) TestRunContainer(c *C) {
 542         s.executor.runFunc = func() {
 543                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 544                 s.executor.exit <- 0
 545         }
 546
 547         var logs TestLogs
 548         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 549         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 550         s.runner.Container.Command = []string{"./hw"}
 551         s.runner.Container.OutputStorageClasses = []string{"default"}
 552
 553         imageID, err := s.runner.LoadImage()
 554         c.Assert(err, IsNil)
 555
 556         err = s.runner.CreateContainer(imageID, nil)
 557         c.Assert(err, IsNil)
 558
 559         err = s.runner.StartContainer()
 560         c.Assert(err, IsNil)
 561
 562         err = s.runner.WaitFinish()
 563         c.Assert(err, IsNil)
 564
 565         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 566         c.Check(logs.Stderr.String(), Equals, "")
 567 }
 568
 569 func (s *TestSuite) TestCommitLogs(c *C) {
 570         api := &ArvTestClient{}
 571         kc := &KeepTestClient{}
 572         defer kc.Close()
 573         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 574         c.Assert(err, IsNil)
 575         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 576
 577         cr.CrunchLog.Print("Hello world!")
 578         cr.CrunchLog.Print("Goodbye")
 579         cr.finalState = "Complete"
 580
 581         err = cr.CommitLogs()
 582         c.Check(err, IsNil)
 583
 584         c.Check(api.Calls, Equals, 2)
 585         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 586         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 587         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 588         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 589 }
 590
 591 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 592         api := &ArvTestClient{}
 593         kc := &KeepTestClient{}
 594         defer kc.Close()
 595         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 596         c.Assert(err, IsNil)
 597
 598         err = cr.UpdateContainerRunning("")
 599         c.Check(err, IsNil)
 600
 601         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 602 }
 603
 604 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 605         api := &ArvTestClient{}
 606         kc := &KeepTestClient{}
 607         defer kc.Close()
 608         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 609         c.Assert(err, IsNil)
 610
 611         cr.LogsPDH = new(string)
 612         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 613
 614         cr.ExitCode = new(int)
 615         *cr.ExitCode = 42
 616         cr.finalState = "Complete"
 617
 618         err = cr.UpdateContainerFinal()
 619         c.Check(err, IsNil)
 620
 621         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 622         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 624 }
 625
 626 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 627         api := &ArvTestClient{}
 628         kc := &KeepTestClient{}
 629         defer kc.Close()
 630         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 631         c.Assert(err, IsNil)
 632         cr.cCancelled = true
 633         cr.finalState = "Cancelled"
 634
 635         err = cr.UpdateContainerFinal()
 636         c.Check(err, IsNil)
 637
 638         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 639         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 640         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 641 }
 642
 643 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 644 // dress rehearsal of the Run() function, starting from a JSON container record.
 645 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 646         err := json.Unmarshal([]byte(record), &s.api.Container)
 647         c.Assert(err, IsNil)
 648         initialState := s.api.Container.State
 649
 650         var sm struct {
 651                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 652         }
 653         err = json.Unmarshal([]byte(record), &sm)
 654         c.Check(err, IsNil)
 655         secretMounts, err := json.Marshal(sm)
 656         c.Assert(err, IsNil)
 657         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 658
 659         s.executor.runFunc = func() {
 660                 fn()
 661                 s.executor.exit <- exitCode
 662         }
 663
 664         s.runner.statInterval = 100 * time.Millisecond
 665         s.runner.containerWatchdogInterval = time.Second
 666
 667         realTemp := c.MkDir()
 668         tempcount := 0
 669         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 670                 tempcount++
 671                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 672                 err := os.Mkdir(d, os.ModePerm)
 673                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 674                         // Test case must have pre-populated the tempdir
 675                         err = nil
 676                 }
 677                 return d, err
 678         }
 679         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 680                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 681         }
 682
 683         if extraMounts != nil && len(extraMounts) > 0 {
 684                 err := s.runner.SetupArvMountPoint("keep")
 685                 c.Check(err, IsNil)
 686
 687                 for _, m := range extraMounts {
 688                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 689                 }
 690         }
 691
 692         err = s.runner.Run()
 693         if s.api.CalledWith("container.state", "Complete") != nil {
 694                 c.Check(err, IsNil)
 695         }
 696         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 697                 c.Check(s.api.WasSetRunning, Equals, true)
 698                 var lastupdate arvadosclient.Dict
 699                 for _, content := range s.api.Content {
 700                         if content["container"] != nil {
 701                                 lastupdate = content["container"].(arvadosclient.Dict)
 702                         }
 703                 }
 704                 if lastupdate["log"] == nil {
 705                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 706                 }
 707         }
 708
 709         if err != nil {
 710                 for k, v := range s.api.Logs {
 711                         c.Log(k)
 712                         c.Log(v.String())
 713                 }
 714         }
 715
 716         return s.api, s.runner, realTemp
 717 }
 718
 719 func (s *TestSuite) TestFullRunHello(c *C) {
 720         s.runner.enableMemoryLimit = true
 721         s.runner.networkMode = "default"
 722         s.fullRunHelper(c, `{
 723     "command": ["echo", "hello world"],
 724     "container_image": "`+arvadostest.DockerImage112PDH+`",
 725     "cwd": ".",
 726     "environment": {"foo":"bar","baz":"waz"},
 727     "mounts": {"/tmp": {"kind": "tmp"} },
 728     "output_path": "/tmp",
 729     "priority": 1,
 730     "runtime_constraints": {"vcpus":1,"ram":1000000},
 731     "state": "Locked",
 732     "output_storage_classes": ["default"]
 733 }`, nil, 0, func() {
 734                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 735                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 736                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 737                 c.Check(s.executor.created.VCPUs, Equals, 1)
 738                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 739                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 740                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 741                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 742                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 743         })
 744
 745         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 746         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 747         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 748         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 749         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 750 }
 751
 752 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 753         var ran bool
 754         s.fullRunHelper(c, `{
 755     "command": ["sleep", "3"],
 756     "container_image": "`+arvadostest.DockerImage112PDH+`",
 757     "cwd": ".",
 758     "environment": {},
 759     "mounts": {"/tmp": {"kind": "tmp"} },
 760     "output_path": "/tmp",
 761     "priority": 1,
 762     "runtime_constraints": {},
 763     "scheduling_parameters":{"max_run_time": 1},
 764     "state": "Running"
 765 }`, nil, 2, func() {
 766                 ran = true
 767         })
 768         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 769         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 770         c.Check(ran, Equals, false)
 771 }
 772
 773 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 774         s.fullRunHelper(c, `{
 775     "command": ["sleep", "3"],
 776     "container_image": "`+arvadostest.DockerImage112PDH+`",
 777     "cwd": ".",
 778     "environment": {},
 779     "mounts": {"/tmp": {"kind": "tmp"} },
 780     "output_path": "/tmp",
 781     "priority": 1,
 782     "runtime_constraints": {},
 783     "scheduling_parameters":{"max_run_time": 1},
 784     "state": "Locked"
 785 }`, nil, 0, func() {
 786                 time.Sleep(3 * time.Second)
 787         })
 788
 789         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 790         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 791 }
 792
 793 func (s *TestSuite) TestContainerWaitFails(c *C) {
 794         s.fullRunHelper(c, `{
 795     "command": ["sleep", "3"],
 796     "container_image": "`+arvadostest.DockerImage112PDH+`",
 797     "cwd": ".",
 798     "mounts": {"/tmp": {"kind": "tmp"} },
 799     "output_path": "/tmp",
 800     "priority": 1,
 801     "state": "Locked"
 802 }`, nil, 0, func() {
 803                 s.executor.waitErr = errors.New("Container is not running")
 804         })
 805
 806         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 807         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 808 }
 809
 810 func (s *TestSuite) TestCrunchstat(c *C) {
 811         s.fullRunHelper(c, `{
 812                 "command": ["sleep", "1"],
 813                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 814                 "cwd": ".",
 815                 "environment": {},
 816                 "mounts": {"/tmp": {"kind": "tmp"} },
 817                 "output_path": "/tmp",
 818                 "priority": 1,
 819                 "runtime_constraints": {},
 820                 "state": "Locked"
 821         }`, nil, 0, func() {
 822                 time.Sleep(time.Second)
 823         })
 824
 825         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 826         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 827
 828         // We didn't actually start a container, so crunchstat didn't
 829         // find accounting files and therefore didn't log any stats.
 830         // It should have logged a "can't find accounting files"
 831         // message after one poll interval, though, so we can confirm
 832         // it's alive:
 833         c.Assert(s.api.Logs["crunchstat"], NotNil)
 834         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 835
 836         // The "files never appeared" log assures us that we called
 837         // (*crunchstat.Reporter)Stop(), and that we set it up with
 838         // the correct container ID "abcde":
 839         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 840 }
 841
 842 func (s *TestSuite) TestNodeInfoLog(c *C) {
 843         os.Setenv("SLURMD_NODENAME", "compute2")
 844         s.fullRunHelper(c, `{
 845                 "command": ["sleep", "1"],
 846                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 847                 "cwd": ".",
 848                 "environment": {},
 849                 "mounts": {"/tmp": {"kind": "tmp"} },
 850                 "output_path": "/tmp",
 851                 "priority": 1,
 852                 "runtime_constraints": {},
 853                 "state": "Locked"
 854         }`, nil, 0,
 855                 func() {
 856                         time.Sleep(time.Second)
 857                 })
 858
 859         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 860         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 861
 862         c.Assert(s.api.Logs["node"], NotNil)
 863         json := s.api.Logs["node"].String()
 864         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 865         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 866         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 867
 868         c.Assert(s.api.Logs["node-info"], NotNil)
 869         json = s.api.Logs["node-info"].String()
 870         c.Check(json, Matches, `(?ms).*Host Information.*`)
 871         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 872         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 873         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 874         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 875 }
 876
 877 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 878         s.fullRunHelper(c, `{
 879                 "command": ["sleep", "1"],
 880                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 881                 "cwd": ".",
 882                 "environment": {},
 883                 "mounts": {"/tmp": {"kind": "tmp"} },
 884                 "output_path": "/tmp",
 885                 "priority": 1,
 886                 "runtime_constraints": {},
 887                 "state": "Locked"
 888         }`, nil, 0,
 889                 func() {
 890                 })
 891
 892         c.Assert(s.api.Logs["crunch-run"], NotNil)
 893         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 894         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 895         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`)
 896         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`)
 897 }
 898
 899 func (s *TestSuite) TestCommitNodeInfoBeforeStart(c *C) {
 900         var collection_create, container_update arvadosclient.Dict
 901         s.fullRunHelper(c, `{
 902                 "command": ["true"],
 903                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 904                 "cwd": ".",
 905                 "environment": {},
 906                 "mounts": {"/tmp": {"kind": "tmp"} },
 907                 "output_path": "/tmp",
 908                 "priority": 1,
 909                 "runtime_constraints": {},
 910                 "state": "Locked",
 911                 "uuid": "zzzzz-dz642-202301121543210"
 912         }`, nil, 0,
 913                 func() {
 914                         collection_create = s.api.CalledWith("ensure_unique_name", true)
 915                         container_update = s.api.CalledWith("container.state", "Running")
 916                 })
 917
 918         c.Assert(collection_create, NotNil)
 919         log_collection := collection_create["collection"].(arvadosclient.Dict)
 920         c.Check(log_collection["name"], Equals, "logs for zzzzz-dz642-202301121543210")
 921         manifest_text := log_collection["manifest_text"].(string)
 922         // We check that the file size is at least two digits as an easy way to
 923         // check the file isn't empty.
 924         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node-info\.txt( .+)?\n`)
 925         c.Check(manifest_text, Matches, `\. .+ \d+:\d{2,}:node\.json( .+)?\n`)
 926
 927         c.Assert(container_update, NotNil)
 928         // As of Arvados 2.5.0, the container update must specify its log in PDH
 929         // format for the API server to propagate it to container requests, which
 930         // is what we care about for this test.
 931         expect_pdh := fmt.Sprintf("%x+%d", md5.Sum([]byte(manifest_text)), len(manifest_text))
 932         c.Check(container_update["container"].(arvadosclient.Dict)["log"], Equals, expect_pdh)
 933 }
 934
 935 func (s *TestSuite) TestContainerRecordLog(c *C) {
 936         s.fullRunHelper(c, `{
 937                 "command": ["sleep", "1"],
 938                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 939                 "cwd": ".",
 940                 "environment": {},
 941                 "mounts": {"/tmp": {"kind": "tmp"} },
 942                 "output_path": "/tmp",
 943                 "priority": 1,
 944                 "runtime_constraints": {},
 945                 "state": "Locked"
 946         }`, nil, 0,
 947                 func() {
 948                         time.Sleep(time.Second)
 949                 })
 950
 951         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 952         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 953
 954         c.Assert(s.api.Logs["container"], NotNil)
 955         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 956 }
 957
 958 func (s *TestSuite) TestFullRunStderr(c *C) {
 959         s.fullRunHelper(c, `{
 960     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 961     "container_image": "`+arvadostest.DockerImage112PDH+`",
 962     "cwd": ".",
 963     "environment": {},
 964     "mounts": {"/tmp": {"kind": "tmp"} },
 965     "output_path": "/tmp",
 966     "priority": 1,
 967     "runtime_constraints": {},
 968     "state": "Locked"
 969 }`, nil, 1, func() {
 970                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 971                 fmt.Fprintln(s.executor.created.Stderr, "world")
 972         })
 973
 974         final := s.api.CalledWith("container.state", "Complete")
 975         c.Assert(final, NotNil)
 976         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 977         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 978
 979         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 980         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 981 }
 982
 983 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 984         s.fullRunHelper(c, `{
 985     "command": ["pwd"],
 986     "container_image": "`+arvadostest.DockerImage112PDH+`",
 987     "cwd": ".",
 988     "environment": {},
 989     "mounts": {"/tmp": {"kind": "tmp"} },
 990     "output_path": "/tmp",
 991     "priority": 1,
 992     "runtime_constraints": {},
 993     "state": "Locked"
 994 }`, nil, 0, func() {
 995                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 996         })
 997
 998         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 999         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1000         c.Log(s.api.Logs["stdout"])
1001         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
1002 }
1003
1004 func (s *TestSuite) TestFullRunSetCwd(c *C) {
1005         s.fullRunHelper(c, `{
1006     "command": ["pwd"],
1007     "container_image": "`+arvadostest.DockerImage112PDH+`",
1008     "cwd": "/bin",
1009     "environment": {},
1010     "mounts": {"/tmp": {"kind": "tmp"} },
1011     "output_path": "/tmp",
1012     "priority": 1,
1013     "runtime_constraints": {},
1014     "state": "Locked"
1015 }`, nil, 0, func() {
1016                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1017         })
1018
1019         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1020         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1021         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1022 }
1023
1024 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
1025         s.fullRunHelper(c, `{
1026     "command": ["pwd"],
1027     "container_image": "`+arvadostest.DockerImage112PDH+`",
1028     "cwd": "/bin",
1029     "environment": {},
1030     "mounts": {"/tmp": {"kind": "tmp"} },
1031     "output_path": "/tmp",
1032     "priority": 1,
1033     "runtime_constraints": {},
1034     "state": "Locked",
1035     "output_storage_classes": ["foo", "bar"]
1036 }`, nil, 0, func() {
1037                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
1038         })
1039
1040         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1041         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1042         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1043         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1044         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1045 }
1046
1047 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1048         s.fullRunHelper(c, `{
1049     "command": ["pwd"],
1050     "container_image": "`+arvadostest.DockerImage112PDH+`",
1051     "cwd": "/bin",
1052     "environment": {},
1053     "mounts": {"/tmp": {"kind": "tmp"} },
1054     "output_path": "/tmp",
1055     "priority": 1,
1056     "runtime_constraints": {"cuda": {"device_count": 2}},
1057     "state": "Locked",
1058     "output_storage_classes": ["foo", "bar"]
1059 }`, nil, 0, func() {
1060                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1061         })
1062         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1063 }
1064
1065 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1066         s.fullRunHelper(c, `{
1067     "command": ["pwd"],
1068     "container_image": "`+arvadostest.DockerImage112PDH+`",
1069     "cwd": "/bin",
1070     "environment": {},
1071     "mounts": {"/tmp": {"kind": "tmp"} },
1072     "output_path": "/tmp",
1073     "priority": 1,
1074     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1075     "state": "Locked",
1076     "output_storage_classes": ["foo", "bar"]
1077 }`, nil, 0, func() {
1078                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1079         })
1080         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1081 }
1082
1083 func (s *TestSuite) TestStopOnSignal(c *C) {
1084         s.executor.runFunc = func() {
1085                 s.executor.created.Stdout.Write([]byte("foo\n"))
1086                 s.runner.SigChan <- syscall.SIGINT
1087         }
1088         s.testStopContainer(c)
1089 }
1090
1091 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1092         s.executor.runFunc = func() {
1093                 s.executor.created.Stdout.Write([]byte("foo\n"))
1094                 s.runner.ArvMountExit <- nil
1095                 close(s.runner.ArvMountExit)
1096         }
1097         s.runner.ArvMountExit = make(chan error)
1098         s.testStopContainer(c)
1099 }
1100
1101 func (s *TestSuite) testStopContainer(c *C) {
1102         record := `{
1103     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1104     "container_image": "` + arvadostest.DockerImage112PDH + `",
1105     "cwd": ".",
1106     "environment": {},
1107     "mounts": {"/tmp": {"kind": "tmp"} },
1108     "output_path": "/tmp",
1109     "priority": 1,
1110     "runtime_constraints": {},
1111     "state": "Locked"
1112 }`
1113
1114         err := json.Unmarshal([]byte(record), &s.api.Container)
1115         c.Assert(err, IsNil)
1116
1117         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1118         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1119                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1120         }
1121
1122         done := make(chan error)
1123         go func() {
1124                 done <- s.runner.Run()
1125         }()
1126         select {
1127         case <-time.After(20 * time.Second):
1128                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1129                 c.Fatal("timed out")
1130         case err = <-done:
1131                 c.Check(err, IsNil)
1132         }
1133         for k, v := range s.api.Logs {
1134                 c.Log(k)
1135                 c.Log(v.String(), "\n")
1136         }
1137
1138         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1139         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1140         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1141 }
1142
1143 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1144         s.fullRunHelper(c, `{
1145     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1146     "container_image": "`+arvadostest.DockerImage112PDH+`",
1147     "cwd": "/bin",
1148     "environment": {"FROBIZ": "bilbo"},
1149     "mounts": {"/tmp": {"kind": "tmp"} },
1150     "output_path": "/tmp",
1151     "priority": 1,
1152     "runtime_constraints": {},
1153     "state": "Locked"
1154 }`, nil, 0, func() {
1155                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1156         })
1157
1158         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1159         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1160         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1161 }
1162
1163 type ArvMountCmdLine struct {
1164         Cmd   []string
1165         token string
1166 }
1167
1168 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1169         am.Cmd = c
1170         am.token = token
1171         return nil, nil
1172 }
1173
1174 func stubCert(temp string) string {
1175         path := temp + "/ca-certificates.crt"
1176         crt, _ := os.Create(path)
1177         crt.Close()
1178         arvadosclient.CertFiles = []string{path}
1179         return path
1180 }
1181
1182 func (s *TestSuite) TestSetupMounts(c *C) {
1183         cr := s.runner
1184         am := &ArvMountCmdLine{}
1185         cr.RunArvMount = am.ArvMountTest
1186         cr.ContainerArvClient = &ArvTestClient{}
1187         cr.ContainerKeepClient = &KeepTestClient{}
1188         cr.Container.OutputStorageClasses = []string{"default"}
1189
1190         realTemp := c.MkDir()
1191         certTemp := c.MkDir()
1192         stubCertPath := stubCert(certTemp)
1193         cr.parentTemp = realTemp
1194
1195         i := 0
1196         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1197                 i++
1198                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1199                 err := os.Mkdir(d, os.ModePerm)
1200                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1201                         // Test case must have pre-populated the tempdir
1202                         err = nil
1203                 }
1204                 return d, err
1205         }
1206
1207         checkEmpty := func() {
1208                 // Should be deleted.
1209                 _, err := os.Stat(realTemp)
1210                 c.Assert(os.IsNotExist(err), Equals, true)
1211
1212                 // Now recreate it for the next test.
1213                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1214         }
1215
1216         {
1217                 i = 0
1218                 cr.ArvMountPoint = ""
1219                 cr.Container.Mounts = make(map[string]arvados.Mount)
1220                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1221                 cr.Container.OutputPath = "/tmp"
1222                 cr.statInterval = 5 * time.Second
1223                 bindmounts, err := cr.SetupMounts()
1224                 c.Check(err, IsNil)
1225                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1226                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1227                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1228                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1229                 os.RemoveAll(cr.ArvMountPoint)
1230                 cr.CleanupDirs()
1231                 checkEmpty()
1232         }
1233
1234         {
1235                 i = 0
1236                 cr.ArvMountPoint = ""
1237                 cr.Container.Mounts = make(map[string]arvados.Mount)
1238                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1239                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1240                 cr.Container.OutputPath = "/out"
1241                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1242
1243                 bindmounts, err := cr.SetupMounts()
1244                 c.Check(err, IsNil)
1245                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1246                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1247                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1248                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1249                 os.RemoveAll(cr.ArvMountPoint)
1250                 cr.CleanupDirs()
1251                 checkEmpty()
1252         }
1253
1254         {
1255                 i = 0
1256                 cr.ArvMountPoint = ""
1257                 cr.Container.Mounts = make(map[string]arvados.Mount)
1258                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1259                 cr.Container.OutputPath = "/tmp"
1260                 cr.Container.RuntimeConstraints.API = true
1261                 cr.Container.OutputStorageClasses = []string{"default"}
1262
1263                 bindmounts, err := cr.SetupMounts()
1264                 c.Check(err, IsNil)
1265                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1266                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1267                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1268                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1269                 os.RemoveAll(cr.ArvMountPoint)
1270                 cr.CleanupDirs()
1271                 checkEmpty()
1272
1273                 cr.Container.RuntimeConstraints.API = false
1274         }
1275
1276         {
1277                 i = 0
1278                 cr.ArvMountPoint = ""
1279                 cr.Container.Mounts = map[string]arvados.Mount{
1280                         "/keeptmp": {Kind: "collection", Writable: true},
1281                 }
1282                 cr.Container.OutputPath = "/keeptmp"
1283
1284                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1285
1286                 bindmounts, err := cr.SetupMounts()
1287                 c.Check(err, IsNil)
1288                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1289                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1290                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1291                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1292                 os.RemoveAll(cr.ArvMountPoint)
1293                 cr.CleanupDirs()
1294                 checkEmpty()
1295         }
1296
1297         {
1298                 i = 0
1299                 cr.ArvMountPoint = ""
1300                 cr.Container.Mounts = map[string]arvados.Mount{
1301                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1302                         "/keepout": {Kind: "collection", Writable: true},
1303                 }
1304                 cr.Container.OutputPath = "/keepout"
1305
1306                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1307                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1308
1309                 bindmounts, err := cr.SetupMounts()
1310                 c.Check(err, IsNil)
1311                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1312                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1313                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1314                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1315                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1316                         "/keepout": {realTemp + "/keep1/tmp0", false},
1317                 })
1318                 os.RemoveAll(cr.ArvMountPoint)
1319                 cr.CleanupDirs()
1320                 checkEmpty()
1321         }
1322
1323         {
1324                 i = 0
1325                 cr.ArvMountPoint = ""
1326                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1327                 cr.Container.Mounts = map[string]arvados.Mount{
1328                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1329                         "/keepout": {Kind: "collection", Writable: true},
1330                 }
1331                 cr.Container.OutputPath = "/keepout"
1332
1333                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1334                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1335
1336                 bindmounts, err := cr.SetupMounts()
1337                 c.Check(err, IsNil)
1338                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1339                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1340                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1341                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1342                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1343                         "/keepout": {realTemp + "/keep1/tmp0", false},
1344                 })
1345                 os.RemoveAll(cr.ArvMountPoint)
1346                 cr.CleanupDirs()
1347                 checkEmpty()
1348         }
1349
1350         for _, test := range []struct {
1351                 in  interface{}
1352                 out string
1353         }{
1354                 {in: "foo", out: `"foo"`},
1355                 {in: nil, out: `null`},
1356                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1357         } {
1358                 i = 0
1359                 cr.ArvMountPoint = ""
1360                 cr.Container.Mounts = map[string]arvados.Mount{
1361                         "/mnt/test.json": {Kind: "json", Content: test.in},
1362                 }
1363                 bindmounts, err := cr.SetupMounts()
1364                 c.Check(err, IsNil)
1365                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1366                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1367                 })
1368                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1369                 c.Check(err, IsNil)
1370                 c.Check(content, DeepEquals, []byte(test.out))
1371                 os.RemoveAll(cr.ArvMountPoint)
1372                 cr.CleanupDirs()
1373                 checkEmpty()
1374         }
1375
1376         for _, test := range []struct {
1377                 in  interface{}
1378                 out string
1379         }{
1380                 {in: "foo", out: `foo`},
1381                 {in: nil, out: "error"},
1382                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1383         } {
1384                 i = 0
1385                 cr.ArvMountPoint = ""
1386                 cr.Container.Mounts = map[string]arvados.Mount{
1387                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1388                 }
1389                 bindmounts, err := cr.SetupMounts()
1390                 if test.out == "error" {
1391                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1392                 } else {
1393                         c.Check(err, IsNil)
1394                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1395                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1396                         })
1397                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1398                         c.Check(err, IsNil)
1399                         c.Check(content, DeepEquals, []byte(test.out))
1400                 }
1401                 os.RemoveAll(cr.ArvMountPoint)
1402                 cr.CleanupDirs()
1403                 checkEmpty()
1404         }
1405
1406         // Read-only mount points are allowed underneath output_dir mount point
1407         {
1408                 i = 0
1409                 cr.ArvMountPoint = ""
1410                 cr.Container.Mounts = make(map[string]arvados.Mount)
1411                 cr.Container.Mounts = map[string]arvados.Mount{
1412                         "/tmp":     {Kind: "tmp"},
1413                         "/tmp/foo": {Kind: "collection"},
1414                 }
1415                 cr.Container.OutputPath = "/tmp"
1416
1417                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1418
1419                 bindmounts, err := cr.SetupMounts()
1420                 c.Check(err, IsNil)
1421                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1422                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5", "--ram-cache",
1423                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1424                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1425                         "/tmp":     {realTemp + "/tmp2", false},
1426                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1427                 })
1428                 os.RemoveAll(cr.ArvMountPoint)
1429                 cr.CleanupDirs()
1430                 checkEmpty()
1431         }
1432
1433         // Writable mount points copied to output_dir mount point
1434         {
1435                 i = 0
1436                 cr.ArvMountPoint = ""
1437                 cr.Container.Mounts = make(map[string]arvados.Mount)
1438                 cr.Container.Mounts = map[string]arvados.Mount{
1439                         "/tmp": {Kind: "tmp"},
1440                         "/tmp/foo": {Kind: "collection",
1441                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1442                                 Writable:         true},
1443                         "/tmp/bar": {Kind: "collection",
1444                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1445                                 Path:             "baz",
1446                                 Writable:         true},
1447                 }
1448                 cr.Container.OutputPath = "/tmp"
1449
1450                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1451                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1452
1453                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1454                 rf.Write([]byte("bar"))
1455                 rf.Close()
1456
1457                 _, err := cr.SetupMounts()
1458                 c.Check(err, IsNil)
1459                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1460                 c.Check(err, IsNil)
1461                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1462                 c.Check(err, IsNil)
1463                 os.RemoveAll(cr.ArvMountPoint)
1464                 cr.CleanupDirs()
1465                 checkEmpty()
1466         }
1467
1468         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1469         {
1470                 i = 0
1471                 cr.ArvMountPoint = ""
1472                 cr.Container.Mounts = make(map[string]arvados.Mount)
1473                 cr.Container.Mounts = map[string]arvados.Mount{
1474                         "/tmp":     {Kind: "tmp"},
1475                         "/tmp/foo": {Kind: "tmp"},
1476                 }
1477                 cr.Container.OutputPath = "/tmp"
1478
1479                 _, err := cr.SetupMounts()
1480                 c.Check(err, NotNil)
1481                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1482                 os.RemoveAll(cr.ArvMountPoint)
1483                 cr.CleanupDirs()
1484                 checkEmpty()
1485         }
1486
1487         // Only mount point of kind 'collection' is allowed for stdin
1488         {
1489                 i = 0
1490                 cr.ArvMountPoint = ""
1491                 cr.Container.Mounts = make(map[string]arvados.Mount)
1492                 cr.Container.Mounts = map[string]arvados.Mount{
1493                         "stdin": {Kind: "tmp"},
1494                 }
1495
1496                 _, err := cr.SetupMounts()
1497                 c.Check(err, NotNil)
1498                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1499                 os.RemoveAll(cr.ArvMountPoint)
1500                 cr.CleanupDirs()
1501                 checkEmpty()
1502         }
1503
1504         // git_tree mounts
1505         {
1506                 i = 0
1507                 cr.ArvMountPoint = ""
1508                 (*GitMountSuite)(nil).useTestGitServer(c)
1509                 cr.token = arvadostest.ActiveToken
1510                 cr.Container.Mounts = make(map[string]arvados.Mount)
1511                 cr.Container.Mounts = map[string]arvados.Mount{
1512                         "/tip": {
1513                                 Kind:   "git_tree",
1514                                 UUID:   arvadostest.Repository2UUID,
1515                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1516                                 Path:   "/",
1517                         },
1518                         "/non-tip": {
1519                                 Kind:   "git_tree",
1520                                 UUID:   arvadostest.Repository2UUID,
1521                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1522                                 Path:   "/",
1523                         },
1524                 }
1525                 cr.Container.OutputPath = "/tmp"
1526
1527                 bindmounts, err := cr.SetupMounts()
1528                 c.Check(err, IsNil)
1529
1530                 for path, mount := range bindmounts {
1531                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1532                 }
1533
1534                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1535                 c.Check(err, IsNil)
1536                 c.Check(string(data), Equals, "\000\001\002\003")
1537                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1538                 c.Check(err, FitsTypeOf, &os.PathError{})
1539                 c.Check(os.IsNotExist(err), Equals, true)
1540
1541                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1542                 c.Check(err, IsNil)
1543                 c.Check(string(data), Equals, "\000\001\002\003")
1544                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1545                 c.Check(err, IsNil)
1546                 c.Check(string(data), Equals, "testfile\n")
1547
1548                 cr.CleanupDirs()
1549                 checkEmpty()
1550         }
1551 }
1552
1553 func (s *TestSuite) TestStdout(c *C) {
1554         helperRecord := `{
1555                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1556                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1557                 "cwd": "/bin",
1558                 "environment": {"FROBIZ": "bilbo"},
1559                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1560                 "output_path": "/tmp",
1561                 "priority": 1,
1562                 "runtime_constraints": {},
1563                 "state": "Locked"
1564         }`
1565
1566         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1567                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1568         })
1569
1570         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1571         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1572         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1573 }
1574
1575 // Used by the TestStdoutWithWrongPath*()
1576 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1577         err := json.Unmarshal([]byte(record), &s.api.Container)
1578         c.Assert(err, IsNil)
1579         s.executor.runFunc = fn
1580         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1581         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1582                 return s.api, &KeepTestClient{}, nil, nil
1583         }
1584         return s.api, s.runner, s.runner.Run()
1585 }
1586
1587 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1588         _, _, err := s.stdoutErrorRunHelper(c, `{
1589     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1590     "output_path": "/tmp",
1591     "state": "Locked"
1592 }`, func() {})
1593         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1594 }
1595
1596 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1597         _, _, err := s.stdoutErrorRunHelper(c, `{
1598     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1599     "output_path": "/tmp",
1600     "state": "Locked"
1601 }`, func() {})
1602         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1603 }
1604
1605 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1606         _, _, err := s.stdoutErrorRunHelper(c, `{
1607     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1608     "output_path": "/tmp",
1609     "state": "Locked"
1610 }`, func() {})
1611         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1612 }
1613
1614 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1615         s.fullRunHelper(c, `{
1616     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1617     "container_image": "`+arvadostest.DockerImage112PDH+`",
1618     "cwd": "/bin",
1619     "environment": {},
1620     "mounts": {"/tmp": {"kind": "tmp"} },
1621     "output_path": "/tmp",
1622     "priority": 1,
1623     "runtime_constraints": {"API": true},
1624     "state": "Locked"
1625 }`, nil, 0, func() {
1626                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1627                 s.executor.exit <- 3
1628         })
1629         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1630         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1631         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1632 }
1633
1634 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1635         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1636         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1637         s.fullRunHelper(c, `{
1638     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1639     "container_image": "`+arvadostest.DockerImage112PDH+`",
1640     "cwd": "/bin",
1641     "environment": {},
1642     "mounts": {"/tmp": {"kind": "tmp"} },
1643     "output_path": "/tmp",
1644     "priority": 1,
1645     "runtime_constraints": {"API": true},
1646     "state": "Locked"
1647 }`, nil, 0, func() {
1648                 s.api.Container.Output = arvadostest.DockerImage112PDH
1649         })
1650
1651         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1652         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1653         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1654 }
1655
1656 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1657         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1658                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1659                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1660                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1661         }
1662         s.executor.runFunc = func() {
1663                 time.Sleep(time.Second)
1664                 s.executor.exit <- 137
1665         }
1666         record := `{
1667     "command": ["sleep", "1"],
1668     "container_image": "` + arvadostest.DockerImage112PDH + `",
1669     "cwd": "/bin",
1670     "environment": {},
1671     "mounts": {"/tmp": {"kind": "tmp"} },
1672     "output_path": "/tmp",
1673     "priority": 1,
1674     "runtime_constraints": {"API": true},
1675     "state": "Locked"
1676 }`
1677         err := json.Unmarshal([]byte(record), &s.api.Container)
1678         c.Assert(err, IsNil)
1679         err = s.runner.Run()
1680         c.Assert(err, IsNil)
1681         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1682         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1683         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1684         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1685         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1686 }
1687
1688 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1689         helperRecord := `{
1690                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1691                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1692                 "cwd": "/bin",
1693                 "environment": {"FROBIZ": "bilbo"},
1694                 "mounts": {
1695         "/tmp": {"kind": "tmp"},
1696         "/tmp/foo": {"kind": "collection",
1697                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1698                      "exclude_from_output": true
1699         },
1700         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1701     },
1702                 "output_path": "/tmp",
1703                 "priority": 1,
1704                 "runtime_constraints": {},
1705                 "state": "Locked"
1706         }`
1707
1708         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1709
1710         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1711                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1712         })
1713
1714         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1715         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1716         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1717 }
1718
1719 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1720         helperRecord := `{
1721                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1722                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1723                 "cwd": "/bin",
1724                 "environment": {"FROBIZ": "bilbo"},
1725                 "mounts": {
1726         "/tmp": {"kind": "tmp"},
1727         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1728         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1729         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1730         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1731         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1732     },
1733                 "output_path": "/tmp",
1734                 "priority": 1,
1735                 "runtime_constraints": {},
1736                 "state": "Locked",
1737                 "uuid": "zzzzz-dz642-202301130848001"
1738         }`
1739
1740         extraMounts := []string{
1741                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1742                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1743                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1744         }
1745
1746         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1747                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1748         })
1749
1750         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1751                 "/tmp":                   {realtemp + "/tmp1", false},
1752                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1753                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1754                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1755                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1756         })
1757
1758         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1759         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1760         output_count := uint(0)
1761         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1762                 if v["collection"] == nil {
1763                         continue
1764                 }
1765                 collection := v["collection"].(arvadosclient.Dict)
1766                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848001" {
1767                         continue
1768                 }
1769                 c.Check(v["ensure_unique_name"], Equals, true)
1770                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1771 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1772 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1773 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1774 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1775 `)
1776                 output_count++
1777         }
1778         c.Check(output_count, Not(Equals), uint(0))
1779 }
1780
1781 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1782         helperRecord := `{
1783                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1784                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1785                 "cwd": "/bin",
1786                 "environment": {"FROBIZ": "bilbo"},
1787                 "mounts": {
1788         "/tmp": {"kind": "tmp"},
1789         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1790         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1791     },
1792                 "output_path": "/tmp",
1793                 "priority": 1,
1794                 "runtime_constraints": {},
1795                 "state": "Locked",
1796                 "uuid": "zzzzz-dz642-202301130848002"
1797         }`
1798
1799         extraMounts := []string{
1800                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1801         }
1802
1803         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1804                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1805         })
1806
1807         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1808         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1809         output_count := uint(0)
1810         for _, v := range s.runner.ContainerArvClient.(*ArvTestClient).Content {
1811                 if v["collection"] == nil {
1812                         continue
1813                 }
1814                 collection := v["collection"].(arvadosclient.Dict)
1815                 if collection["name"].(string) != "output for zzzzz-dz642-202301130848002" {
1816                         continue
1817                 }
1818                 c.Check(collection["manifest_text"].(string), Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1819 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1820 `)
1821                 output_count++
1822         }
1823         c.Check(output_count, Not(Equals), uint(0))
1824 }
1825
1826 func (s *TestSuite) TestOutputError(c *C) {
1827         helperRecord := `{
1828                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1829                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1830                 "cwd": "/bin",
1831                 "environment": {"FROBIZ": "bilbo"},
1832                 "mounts": {
1833                         "/tmp": {"kind": "tmp"}
1834                 },
1835                 "output_path": "/tmp",
1836                 "priority": 1,
1837                 "runtime_constraints": {},
1838                 "state": "Locked"
1839         }`
1840         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1841                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1842         })
1843
1844         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1845 }
1846
1847 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1848         helperRecord := `{
1849                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1850                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1851                 "cwd": "/bin",
1852                 "environment": {"FROBIZ": "bilbo"},
1853                 "mounts": {
1854         "/tmp": {"kind": "tmp"},
1855         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1856         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1857     },
1858                 "output_path": "/tmp",
1859                 "priority": 1,
1860                 "runtime_constraints": {},
1861                 "state": "Locked"
1862         }`
1863
1864         extraMounts := []string{
1865                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1866         }
1867
1868         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1869                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1870         })
1871
1872         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1873         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1874         for _, v := range api.Content {
1875                 if v["collection"] != nil {
1876                         collection := v["collection"].(arvadosclient.Dict)
1877                         if strings.Index(collection["name"].(string), "output") == 0 {
1878                                 manifest := collection["manifest_text"].(string)
1879                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1880 `)
1881                         }
1882                 }
1883         }
1884 }
1885
1886 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1887         helperRecord := `{
1888                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1889                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1890                 "cwd": "/bin",
1891                 "environment": {"FROBIZ": "bilbo"},
1892                 "mounts": {
1893         "/tmp": {"kind": "tmp"},
1894         "stdin": {"kind": "json", "content": "foo"},
1895         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1896     },
1897                 "output_path": "/tmp",
1898                 "priority": 1,
1899                 "runtime_constraints": {},
1900                 "state": "Locked"
1901         }`
1902
1903         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1904                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1905         })
1906
1907         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1908         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1909         for _, v := range api.Content {
1910                 if v["collection"] != nil {
1911                         collection := v["collection"].(arvadosclient.Dict)
1912                         if strings.Index(collection["name"].(string), "output") == 0 {
1913                                 manifest := collection["manifest_text"].(string)
1914                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1915 `)
1916                         }
1917                 }
1918         }
1919 }
1920
1921 func (s *TestSuite) TestStderrMount(c *C) {
1922         api, cr, _ := s.fullRunHelper(c, `{
1923     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1924     "container_image": "`+arvadostest.DockerImage112PDH+`",
1925     "cwd": ".",
1926     "environment": {},
1927     "mounts": {"/tmp": {"kind": "tmp"},
1928                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1929                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1930     "output_path": "/tmp",
1931     "priority": 1,
1932     "runtime_constraints": {},
1933     "state": "Locked"
1934 }`, nil, 1, func() {
1935                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1936                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1937         })
1938
1939         final := api.CalledWith("container.state", "Complete")
1940         c.Assert(final, NotNil)
1941         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1942         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1943
1944         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1945 }
1946
1947 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1948         s.api.callraw = true
1949         err := s.runner.fetchContainerRecord()
1950         c.Assert(err, IsNil)
1951         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1952         c.Logf("%#v", s.runner.Container)
1953         c.Check(err, IsNil)
1954         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1955 }
1956
1957 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1958         nextState := ""
1959         for _, setup := range []func(){
1960                 func() {
1961                         c.Log("// waitErr = ocl runtime error")
1962                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1963                         nextState = "Cancelled"
1964                 },
1965                 func() {
1966                         c.Log("// loadErr = cannot connect")
1967                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1968                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1969                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1970                         c.Assert(err, IsNil)
1971                         nextState = "Queued"
1972                 },
1973         } {
1974                 s.SetUpTest(c)
1975                 setup()
1976                 s.fullRunHelper(c, `{
1977     "command": ["echo", "hello world"],
1978     "container_image": "`+arvadostest.DockerImage112PDH+`",
1979     "cwd": ".",
1980     "environment": {},
1981     "mounts": {"/tmp": {"kind": "tmp"} },
1982     "output_path": "/tmp",
1983     "priority": 1,
1984     "runtime_constraints": {},
1985     "state": "Locked"
1986 }`, nil, 0, func() {})
1987                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1988                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1989                 if s.runner.brokenNodeHook != "" {
1990                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1991                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1992                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1993                 } else {
1994                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1995                 }
1996         }
1997 }
1998
1999 func (s *TestSuite) TestBadCommand(c *C) {
2000         for _, startError := range []string{
2001                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
2002                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
2003                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
2004         } {
2005                 s.SetUpTest(c)
2006                 s.executor.startErr = errors.New(startError)
2007                 s.fullRunHelper(c, `{
2008     "command": ["echo", "hello world"],
2009     "container_image": "`+arvadostest.DockerImage112PDH+`",
2010     "cwd": ".",
2011     "environment": {},
2012     "mounts": {"/tmp": {"kind": "tmp"} },
2013     "output_path": "/tmp",
2014     "priority": 1,
2015     "runtime_constraints": {},
2016     "state": "Locked"
2017 }`, nil, 0, func() {})
2018                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
2019                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
2020         }
2021 }
2022
2023 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
2024         helperRecord := `{
2025                 "command": ["true"],
2026                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2027                 "cwd": "/bin",
2028                 "mounts": {
2029                     "/tmp": {"kind": "tmp"},
2030                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2031                 },
2032                 "secret_mounts": {
2033                 },
2034                 "output_path": "/tmp",
2035                 "priority": 1,
2036                 "runtime_constraints": {},
2037                 "state": "Locked"
2038         }`
2039
2040         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2041                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2042                 c.Check(err, IsNil)
2043                 c.Check(string(content), Equals, "mypassword")
2044         })
2045
2046         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2047         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2048         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2049         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2050
2051         // under secret mounts, not captured in output
2052         helperRecord = `{
2053                 "command": ["true"],
2054                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2055                 "cwd": "/bin",
2056                 "mounts": {
2057                     "/tmp": {"kind": "tmp"}
2058                 },
2059                 "secret_mounts": {
2060                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2061                 },
2062                 "output_path": "/tmp",
2063                 "priority": 1,
2064                 "runtime_constraints": {},
2065                 "state": "Locked"
2066         }`
2067
2068         s.SetUpTest(c)
2069         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2070                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2071                 c.Check(err, IsNil)
2072                 c.Check(string(content), Equals, "mypassword")
2073         })
2074
2075         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2076         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2077         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2078         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2079
2080         // under secret mounts, output dir is a collection, not captured in output
2081         helperRecord = `{
2082                 "command": ["true"],
2083                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2084                 "cwd": "/bin",
2085                 "mounts": {
2086                     "/tmp": {"kind": "collection", "writable": true}
2087                 },
2088                 "secret_mounts": {
2089                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2090                 },
2091                 "output_path": "/tmp",
2092                 "priority": 1,
2093                 "runtime_constraints": {},
2094                 "state": "Locked"
2095         }`
2096
2097         s.SetUpTest(c)
2098         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2099                 // secret.conf should be provisioned as a separate
2100                 // bind mount, i.e., it should not appear in the
2101                 // (fake) fuse filesystem as viewed from the host.
2102                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2103                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2104                         c.Logf("secret.conf: content %q, err %#v", content, err)
2105                 }
2106                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2107                 c.Check(err, IsNil)
2108         })
2109
2110         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2111         c.Check(err, IsNil)
2112         c.Check(string(content), Equals, "mypassword")
2113         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2114         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2115         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2116         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2117 }
2118
2119 type FakeProcess struct {
2120         cmdLine []string
2121 }
2122
2123 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2124         return fp.cmdLine, nil
2125 }