lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         keepmountTmp             []string
  48         testDispatcherKeepClient KeepTestClient
  49         testContainerKeepClient  KeepTestClient
  50 }
  51
  52 func (s *TestSuite) SetUpTest(c *C) {
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 for i, opt := range cmd {
  66                         if opt == "--mount-tmp" {
  67                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  68                                 if err != nil {
  69                                         return nil, err
  70                                 }
  71                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  72                         }
  73                 }
  74                 return nil, nil
  75         }
  76         s.keepmount = c.MkDir()
  77         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  78         s.keepmountTmp = nil
  79         c.Assert(err, IsNil)
  80         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  81         c.Assert(err, IsNil)
  82         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  83         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  84         c.Assert(err, IsNil)
  85         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  86         c.Assert(err, IsNil)
  87         s.runner.ArvMountPoint = s.keepmount
  88 }
  89
  90 type ArvTestClient struct {
  91         Total   int64
  92         Calls   int
  93         Content []arvadosclient.Dict
  94         arvados.Container
  95         secretMounts []byte
  96         Logs         map[string]*bytes.Buffer
  97         sync.Mutex
  98         WasSetRunning bool
  99         callraw       bool
 100 }
 101
 102 type KeepTestClient struct {
 103         Called         bool
 104         Content        []byte
 105         StorageClasses []string
 106 }
 107
 108 type stubExecutor struct {
 109         imageLoaded bool
 110         loaded      string
 111         loadErr     error
 112         exitCode    int
 113         createErr   error
 114         created     containerSpec
 115         startErr    error
 116         waitSleep   time.Duration
 117         waitErr     error
 118         stopErr     error
 119         stopped     bool
 120         closed      bool
 121         runFunc     func()
 122         exit        chan int
 123 }
 124
 125 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 126         containerClient *arvados.Client) error {
 127         e.loaded = tarball
 128         return e.loadErr
 129 }
 130 func (e *stubExecutor) Runtime() string                 { return "stub" }
 131 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 132 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 133 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 134 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 135 func (e *stubExecutor) Close()                          { e.closed = true }
 136 func (e *stubExecutor) Wait(context.Context) (int, error) {
 137         return <-e.exit, e.waitErr
 138 }
 139 func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _ []string) (*exec.Cmd, error) {
 140         return nil, errors.New("unimplemented")
 141 }
 142 func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
 143
 144 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 145
 146 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 147 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 148 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 149
 150 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 151 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 152
 153 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 154 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 155 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 156 `
 157
 158 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 159
 160 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 161 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 162
 163 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 164 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 165
 166 func (client *ArvTestClient) Create(resourceType string,
 167         parameters arvadosclient.Dict,
 168         output interface{}) error {
 169
 170         client.Mutex.Lock()
 171         defer client.Mutex.Unlock()
 172
 173         client.Calls++
 174         client.Content = append(client.Content, parameters)
 175
 176         if resourceType == "logs" {
 177                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 178                 if client.Logs == nil {
 179                         client.Logs = make(map[string]*bytes.Buffer)
 180                 }
 181                 if client.Logs[et] == nil {
 182                         client.Logs[et] = &bytes.Buffer{}
 183                 }
 184                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 185         }
 186
 187         if resourceType == "collections" && output != nil {
 188                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 189                 outmap := output.(*arvados.Collection)
 190                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 191                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 192         }
 193
 194         return nil
 195 }
 196
 197 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 198         switch {
 199         case method == "GET" && resourceType == "containers" && action == "auth":
 200                 return json.Unmarshal([]byte(`{
 201                         "kind": "arvados#api_client_authorization",
 202                         "uuid": "`+fakeAuthUUID+`",
 203                         "api_token": "`+fakeAuthToken+`"
 204                         }`), output)
 205         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 206                 if client.secretMounts != nil {
 207                         return json.Unmarshal(client.secretMounts, output)
 208                 }
 209                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 210         default:
 211                 return fmt.Errorf("Not found")
 212         }
 213 }
 214
 215 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 216         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 217         var j []byte
 218         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 219                 j = []byte(`{
 220                         "kind": "arvados#nodeList",
 221                         "items": [{
 222                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 223                                 "hostname": "compute2",
 224                                 "properties": {"total_cpu_cores": 16}
 225                         }]}`)
 226         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 227                 if uuid == "" {
 228                         j, err = json.Marshal(map[string]interface{}{
 229                                 "items": []interface{}{client.Container},
 230                                 "kind":  "arvados#nodeList",
 231                         })
 232                 } else {
 233                         j, err = json.Marshal(client.Container)
 234                 }
 235         } else {
 236                 j = []byte(`{
 237                         "command": ["sleep", "1"],
 238                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 239                         "cwd": ".",
 240                         "environment": {},
 241                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 242                         "output_path": "/tmp",
 243                         "priority": 1,
 244                         "runtime_constraints": {}
 245                 }`)
 246         }
 247         return ioutil.NopCloser(bytes.NewReader(j)), err
 248 }
 249
 250 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 251         if resourceType == "collections" {
 252                 if uuid == hwPDH {
 253                         output.(*arvados.Collection).ManifestText = hwManifest
 254                 } else if uuid == otherPDH {
 255                         output.(*arvados.Collection).ManifestText = otherManifest
 256                 } else if uuid == normalizedWithSubdirsPDH {
 257                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 258                 } else if uuid == denormalizedWithSubdirsPDH {
 259                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 260                 }
 261         }
 262         if resourceType == "containers" {
 263                 (*output.(*arvados.Container)) = client.Container
 264         }
 265         return nil
 266 }
 267
 268 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 269         client.Mutex.Lock()
 270         defer client.Mutex.Unlock()
 271         client.Calls++
 272         client.Content = append(client.Content, parameters)
 273         if resourceType == "containers" {
 274                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 275                         client.WasSetRunning = true
 276                 }
 277         } else if resourceType == "collections" {
 278                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 279                 output.(*arvados.Collection).UUID = uuid
 280                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 281         }
 282         return nil
 283 }
 284
 285 var discoveryMap = map[string]interface{}{
 286         "defaultTrashLifetime":               float64(1209600),
 287         "crunchLimitLogBytesPerJob":          float64(67108864),
 288         "crunchLogThrottleBytes":             float64(65536),
 289         "crunchLogThrottlePeriod":            float64(60),
 290         "crunchLogThrottleLines":             float64(1024),
 291         "crunchLogPartialLineThrottlePeriod": float64(5),
 292         "crunchLogBytesPerEvent":             float64(4096),
 293         "crunchLogSecondsBetweenEvents":      float64(1),
 294 }
 295
 296 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 297         return discoveryMap[key], nil
 298 }
 299
 300 // CalledWith returns the parameters from the first API call whose
 301 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 302 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 303 // no call matches, it returns nil.
 304 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 305 call:
 306         for _, content := range client.Content {
 307                 var v interface{} = content
 308                 for _, k := range strings.Split(jpath, ".") {
 309                         if dict, ok := v.(arvadosclient.Dict); !ok {
 310                                 continue call
 311                         } else {
 312                                 v = dict[k]
 313                         }
 314                 }
 315                 if v == expect {
 316                         return content
 317                 }
 318         }
 319         return nil
 320 }
 321
 322 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 323         return locator, nil
 324 }
 325
 326 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 327         client.Content = opts.Data
 328         return arvados.BlockWriteResponse{
 329                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 330         }, nil
 331 }
 332
 333 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 334         return 0, errors.New("not implemented")
 335 }
 336
 337 func (client *KeepTestClient) ClearBlockCache() {
 338 }
 339
 340 func (client *KeepTestClient) Close() {
 341         client.Content = nil
 342 }
 343
 344 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 345         client.StorageClasses = sc
 346 }
 347
 348 type FileWrapper struct {
 349         io.ReadCloser
 350         len int64
 351 }
 352
 353 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 354         return nil, errors.New("not implemented")
 355 }
 356
 357 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 358         return 0, errors.New("not implemented")
 359 }
 360
 361 func (fw FileWrapper) Size() int64 {
 362         return fw.len
 363 }
 364
 365 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 366         return nil, errors.New("not implemented")
 367 }
 368
 369 func (fw FileWrapper) Truncate(int64) error {
 370         return errors.New("not implemented")
 371 }
 372
 373 func (fw FileWrapper) Write([]byte) (int, error) {
 374         return 0, errors.New("not implemented")
 375 }
 376
 377 func (fw FileWrapper) Sync() error {
 378         return errors.New("not implemented")
 379 }
 380
 381 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 382         return nil, errors.New("not implemented")
 383 }
 384
 385 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 386         return errors.New("not implemented")
 387 }
 388
 389 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 390         if filename == hwImageID+".tar" {
 391                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 392                 client.Called = true
 393                 return FileWrapper{rdr, 1321984}, nil
 394         } else if filename == "/file1_in_main.txt" {
 395                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 396                 client.Called = true
 397                 return FileWrapper{rdr, 3}, nil
 398         }
 399         return nil, nil
 400 }
 401
 402 func (s *TestSuite) TestLoadImage(c *C) {
 403         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 404         s.runner.Container.Mounts = map[string]arvados.Mount{
 405                 "/out": {Kind: "tmp", Writable: true},
 406         }
 407         s.runner.Container.OutputPath = "/out"
 408
 409         _, err := s.runner.SetupMounts()
 410         c.Assert(err, IsNil)
 411
 412         imageID, err := s.runner.LoadImage()
 413         c.Check(err, IsNil)
 414         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 415         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 416
 417         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 418         s.executor.imageLoaded = false
 419         s.executor.loaded = ""
 420         s.executor.loadErr = errors.New("bork")
 421         imageID, err = s.runner.LoadImage()
 422         c.Check(err, ErrorMatches, ".*bork")
 423         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 424
 425         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 426         s.executor.imageLoaded = false
 427         s.executor.loaded = ""
 428         s.executor.loadErr = nil
 429         imageID, err = s.runner.LoadImage()
 430         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 431         c.Check(s.executor.loaded, Equals, "")
 432 }
 433
 434 type ArvErrorTestClient struct{}
 435
 436 func (ArvErrorTestClient) Create(resourceType string,
 437         parameters arvadosclient.Dict,
 438         output interface{}) error {
 439         return nil
 440 }
 441
 442 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 443         if method == "GET" && resourceType == "containers" && action == "auth" {
 444                 return nil
 445         }
 446         return errors.New("ArvError")
 447 }
 448
 449 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 450         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 451         return nil, errors.New("ArvError")
 452 }
 453
 454 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 455         return errors.New("ArvError")
 456 }
 457
 458 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 459         return nil
 460 }
 461
 462 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 463         return discoveryMap[key], nil
 464 }
 465
 466 type KeepErrorTestClient struct {
 467         KeepTestClient
 468 }
 469
 470 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 471         return nil, errors.New("KeepError")
 472 }
 473
 474 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 475         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 476 }
 477
 478 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 479         return "", errors.New("KeepError")
 480 }
 481
 482 type KeepReadErrorTestClient struct {
 483         KeepTestClient
 484 }
 485
 486 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 487         return 0, errors.New("KeepError")
 488 }
 489
 490 type ErrorReader struct {
 491         FileWrapper
 492 }
 493
 494 func (ErrorReader) Read(p []byte) (n int, err error) {
 495         return 0, errors.New("ErrorReader")
 496 }
 497
 498 func (ErrorReader) Seek(int64, int) (int64, error) {
 499         return 0, errors.New("ErrorReader")
 500 }
 501
 502 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 503         return ErrorReader{}, nil
 504 }
 505
 506 type ClosableBuffer struct {
 507         bytes.Buffer
 508 }
 509
 510 func (*ClosableBuffer) Close() error {
 511         return nil
 512 }
 513
 514 type TestLogs struct {
 515         Stdout ClosableBuffer
 516         Stderr ClosableBuffer
 517 }
 518
 519 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 520         if logstr == "stdout" {
 521                 return &tl.Stdout, nil
 522         }
 523         if logstr == "stderr" {
 524                 return &tl.Stderr, nil
 525         }
 526         return nil, errors.New("???")
 527 }
 528
 529 func dockerLog(fd byte, msg string) []byte {
 530         by := []byte(msg)
 531         header := make([]byte, 8+len(by))
 532         header[0] = fd
 533         header[7] = byte(len(by))
 534         copy(header[8:], by)
 535         return header
 536 }
 537
 538 func (s *TestSuite) TestRunContainer(c *C) {
 539         s.executor.runFunc = func() {
 540                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 541                 s.executor.exit <- 0
 542         }
 543
 544         var logs TestLogs
 545         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 546         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 547         s.runner.Container.Command = []string{"./hw"}
 548         s.runner.Container.OutputStorageClasses = []string{"default"}
 549
 550         imageID, err := s.runner.LoadImage()
 551         c.Assert(err, IsNil)
 552
 553         err = s.runner.CreateContainer(imageID, nil)
 554         c.Assert(err, IsNil)
 555
 556         err = s.runner.StartContainer()
 557         c.Assert(err, IsNil)
 558
 559         err = s.runner.WaitFinish()
 560         c.Assert(err, IsNil)
 561
 562         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 563         c.Check(logs.Stderr.String(), Equals, "")
 564 }
 565
 566 func (s *TestSuite) TestCommitLogs(c *C) {
 567         api := &ArvTestClient{}
 568         kc := &KeepTestClient{}
 569         defer kc.Close()
 570         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 571         c.Assert(err, IsNil)
 572         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 573
 574         cr.CrunchLog.Print("Hello world!")
 575         cr.CrunchLog.Print("Goodbye")
 576         cr.finalState = "Complete"
 577
 578         err = cr.CommitLogs()
 579         c.Check(err, IsNil)
 580
 581         c.Check(api.Calls, Equals, 2)
 582         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 583         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 584         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 585         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 586 }
 587
 588 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 589         api := &ArvTestClient{}
 590         kc := &KeepTestClient{}
 591         defer kc.Close()
 592         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 593         c.Assert(err, IsNil)
 594
 595         err = cr.UpdateContainerRunning()
 596         c.Check(err, IsNil)
 597
 598         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 599 }
 600
 601 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 602         api := &ArvTestClient{}
 603         kc := &KeepTestClient{}
 604         defer kc.Close()
 605         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 606         c.Assert(err, IsNil)
 607
 608         cr.LogsPDH = new(string)
 609         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 610
 611         cr.ExitCode = new(int)
 612         *cr.ExitCode = 42
 613         cr.finalState = "Complete"
 614
 615         err = cr.UpdateContainerFinal()
 616         c.Check(err, IsNil)
 617
 618         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 619         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 620         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 621 }
 622
 623 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 624         api := &ArvTestClient{}
 625         kc := &KeepTestClient{}
 626         defer kc.Close()
 627         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 628         c.Assert(err, IsNil)
 629         cr.cCancelled = true
 630         cr.finalState = "Cancelled"
 631
 632         err = cr.UpdateContainerFinal()
 633         c.Check(err, IsNil)
 634
 635         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 636         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 637         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 638 }
 639
 640 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 641 // dress rehearsal of the Run() function, starting from a JSON container record.
 642 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 643         err := json.Unmarshal([]byte(record), &s.api.Container)
 644         c.Assert(err, IsNil)
 645         initialState := s.api.Container.State
 646
 647         var sm struct {
 648                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 649         }
 650         err = json.Unmarshal([]byte(record), &sm)
 651         c.Check(err, IsNil)
 652         secretMounts, err := json.Marshal(sm)
 653         c.Assert(err, IsNil)
 654         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 655
 656         s.executor.runFunc = func() {
 657                 fn()
 658                 s.executor.exit <- exitCode
 659         }
 660
 661         s.runner.statInterval = 100 * time.Millisecond
 662         s.runner.containerWatchdogInterval = time.Second
 663
 664         realTemp := c.MkDir()
 665         tempcount := 0
 666         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 667                 tempcount++
 668                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 669                 err := os.Mkdir(d, os.ModePerm)
 670                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 671                         // Test case must have pre-populated the tempdir
 672                         err = nil
 673                 }
 674                 return d, err
 675         }
 676         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 677                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 678         }
 679
 680         if extraMounts != nil && len(extraMounts) > 0 {
 681                 err := s.runner.SetupArvMountPoint("keep")
 682                 c.Check(err, IsNil)
 683
 684                 for _, m := range extraMounts {
 685                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 686                 }
 687         }
 688
 689         err = s.runner.Run()
 690         if s.api.CalledWith("container.state", "Complete") != nil {
 691                 c.Check(err, IsNil)
 692         }
 693         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 694                 c.Check(s.api.WasSetRunning, Equals, true)
 695                 var lastupdate arvadosclient.Dict
 696                 for _, content := range s.api.Content {
 697                         if content["container"] != nil {
 698                                 lastupdate = content["container"].(arvadosclient.Dict)
 699                         }
 700                 }
 701                 if lastupdate["log"] == nil {
 702                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 703                 }
 704         }
 705
 706         if err != nil {
 707                 for k, v := range s.api.Logs {
 708                         c.Log(k)
 709                         c.Log(v.String())
 710                 }
 711         }
 712
 713         return s.api, s.runner, realTemp
 714 }
 715
 716 func (s *TestSuite) TestFullRunHello(c *C) {
 717         s.runner.enableMemoryLimit = true
 718         s.runner.networkMode = "default"
 719         s.fullRunHelper(c, `{
 720     "command": ["echo", "hello world"],
 721     "container_image": "`+arvadostest.DockerImage112PDH+`",
 722     "cwd": ".",
 723     "environment": {"foo":"bar","baz":"waz"},
 724     "mounts": {"/tmp": {"kind": "tmp"} },
 725     "output_path": "/tmp",
 726     "priority": 1,
 727     "runtime_constraints": {"vcpus":1,"ram":1000000},
 728     "state": "Locked",
 729     "output_storage_classes": ["default"]
 730 }`, nil, 0, func() {
 731                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 732                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 733                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 734                 c.Check(s.executor.created.VCPUs, Equals, 1)
 735                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 736                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 737                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 738                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 739                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 740         })
 741
 742         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 743         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 744         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 745         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 746         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 747 }
 748
 749 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 750         var ran bool
 751         s.fullRunHelper(c, `{
 752     "command": ["sleep", "3"],
 753     "container_image": "`+arvadostest.DockerImage112PDH+`",
 754     "cwd": ".",
 755     "environment": {},
 756     "mounts": {"/tmp": {"kind": "tmp"} },
 757     "output_path": "/tmp",
 758     "priority": 1,
 759     "runtime_constraints": {},
 760     "scheduling_parameters":{"max_run_time": 1},
 761     "state": "Running"
 762 }`, nil, 2, func() {
 763                 ran = true
 764         })
 765         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 766         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 767         c.Check(ran, Equals, false)
 768 }
 769
 770 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 771         s.fullRunHelper(c, `{
 772     "command": ["sleep", "3"],
 773     "container_image": "`+arvadostest.DockerImage112PDH+`",
 774     "cwd": ".",
 775     "environment": {},
 776     "mounts": {"/tmp": {"kind": "tmp"} },
 777     "output_path": "/tmp",
 778     "priority": 1,
 779     "runtime_constraints": {},
 780     "scheduling_parameters":{"max_run_time": 1},
 781     "state": "Locked"
 782 }`, nil, 0, func() {
 783                 time.Sleep(3 * time.Second)
 784         })
 785
 786         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 787         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 788 }
 789
 790 func (s *TestSuite) TestContainerWaitFails(c *C) {
 791         s.fullRunHelper(c, `{
 792     "command": ["sleep", "3"],
 793     "container_image": "`+arvadostest.DockerImage112PDH+`",
 794     "cwd": ".",
 795     "mounts": {"/tmp": {"kind": "tmp"} },
 796     "output_path": "/tmp",
 797     "priority": 1,
 798     "state": "Locked"
 799 }`, nil, 0, func() {
 800                 s.executor.waitErr = errors.New("Container is not running")
 801         })
 802
 803         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 804         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 805 }
 806
 807 func (s *TestSuite) TestCrunchstat(c *C) {
 808         s.fullRunHelper(c, `{
 809                 "command": ["sleep", "1"],
 810                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 811                 "cwd": ".",
 812                 "environment": {},
 813                 "mounts": {"/tmp": {"kind": "tmp"} },
 814                 "output_path": "/tmp",
 815                 "priority": 1,
 816                 "runtime_constraints": {},
 817                 "state": "Locked"
 818         }`, nil, 0, func() {
 819                 time.Sleep(time.Second)
 820         })
 821
 822         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 823         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 824
 825         // We didn't actually start a container, so crunchstat didn't
 826         // find accounting files and therefore didn't log any stats.
 827         // It should have logged a "can't find accounting files"
 828         // message after one poll interval, though, so we can confirm
 829         // it's alive:
 830         c.Assert(s.api.Logs["crunchstat"], NotNil)
 831         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 832
 833         // The "files never appeared" log assures us that we called
 834         // (*crunchstat.Reporter)Stop(), and that we set it up with
 835         // the correct container ID "abcde":
 836         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 837 }
 838
 839 func (s *TestSuite) TestNodeInfoLog(c *C) {
 840         os.Setenv("SLURMD_NODENAME", "compute2")
 841         s.fullRunHelper(c, `{
 842                 "command": ["sleep", "1"],
 843                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 844                 "cwd": ".",
 845                 "environment": {},
 846                 "mounts": {"/tmp": {"kind": "tmp"} },
 847                 "output_path": "/tmp",
 848                 "priority": 1,
 849                 "runtime_constraints": {},
 850                 "state": "Locked"
 851         }`, nil, 0,
 852                 func() {
 853                         time.Sleep(time.Second)
 854                 })
 855
 856         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 857         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 858
 859         c.Assert(s.api.Logs["node"], NotNil)
 860         json := s.api.Logs["node"].String()
 861         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 862         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 863         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 864
 865         c.Assert(s.api.Logs["node-info"], NotNil)
 866         json = s.api.Logs["node-info"].String()
 867         c.Check(json, Matches, `(?ms).*Host Information.*`)
 868         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 869         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 870         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 871         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 872 }
 873
 874 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 875         s.fullRunHelper(c, `{
 876                 "command": ["sleep", "1"],
 877                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 878                 "cwd": ".",
 879                 "environment": {},
 880                 "mounts": {"/tmp": {"kind": "tmp"} },
 881                 "output_path": "/tmp",
 882                 "priority": 1,
 883                 "runtime_constraints": {},
 884                 "state": "Locked"
 885         }`, nil, 0,
 886                 func() {
 887                 })
 888
 889         c.Assert(s.api.Logs["crunch-run"], NotNil)
 890         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 891         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 892         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 893 }
 894
 895 func (s *TestSuite) TestContainerRecordLog(c *C) {
 896         s.fullRunHelper(c, `{
 897                 "command": ["sleep", "1"],
 898                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 899                 "cwd": ".",
 900                 "environment": {},
 901                 "mounts": {"/tmp": {"kind": "tmp"} },
 902                 "output_path": "/tmp",
 903                 "priority": 1,
 904                 "runtime_constraints": {},
 905                 "state": "Locked"
 906         }`, nil, 0,
 907                 func() {
 908                         time.Sleep(time.Second)
 909                 })
 910
 911         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 912         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 913
 914         c.Assert(s.api.Logs["container"], NotNil)
 915         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 916 }
 917
 918 func (s *TestSuite) TestFullRunStderr(c *C) {
 919         s.fullRunHelper(c, `{
 920     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 921     "container_image": "`+arvadostest.DockerImage112PDH+`",
 922     "cwd": ".",
 923     "environment": {},
 924     "mounts": {"/tmp": {"kind": "tmp"} },
 925     "output_path": "/tmp",
 926     "priority": 1,
 927     "runtime_constraints": {},
 928     "state": "Locked"
 929 }`, nil, 1, func() {
 930                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 931                 fmt.Fprintln(s.executor.created.Stderr, "world")
 932         })
 933
 934         final := s.api.CalledWith("container.state", "Complete")
 935         c.Assert(final, NotNil)
 936         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 937         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 938
 939         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 940         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 941 }
 942
 943 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 944         s.fullRunHelper(c, `{
 945     "command": ["pwd"],
 946     "container_image": "`+arvadostest.DockerImage112PDH+`",
 947     "cwd": ".",
 948     "environment": {},
 949     "mounts": {"/tmp": {"kind": "tmp"} },
 950     "output_path": "/tmp",
 951     "priority": 1,
 952     "runtime_constraints": {},
 953     "state": "Locked"
 954 }`, nil, 0, func() {
 955                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 956         })
 957
 958         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 959         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 960         c.Log(s.api.Logs["stdout"])
 961         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 962 }
 963
 964 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 965         s.fullRunHelper(c, `{
 966     "command": ["pwd"],
 967     "container_image": "`+arvadostest.DockerImage112PDH+`",
 968     "cwd": "/bin",
 969     "environment": {},
 970     "mounts": {"/tmp": {"kind": "tmp"} },
 971     "output_path": "/tmp",
 972     "priority": 1,
 973     "runtime_constraints": {},
 974     "state": "Locked"
 975 }`, nil, 0, func() {
 976                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 977         })
 978
 979         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 980         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 981         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 982 }
 983
 984 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 985         s.fullRunHelper(c, `{
 986     "command": ["pwd"],
 987     "container_image": "`+arvadostest.DockerImage112PDH+`",
 988     "cwd": "/bin",
 989     "environment": {},
 990     "mounts": {"/tmp": {"kind": "tmp"} },
 991     "output_path": "/tmp",
 992     "priority": 1,
 993     "runtime_constraints": {},
 994     "state": "Locked",
 995     "output_storage_classes": ["foo", "bar"]
 996 }`, nil, 0, func() {
 997                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 998         })
 999
1000         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1001         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1002         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
1003         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1004         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1005 }
1006
1007 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1008         s.fullRunHelper(c, `{
1009     "command": ["pwd"],
1010     "container_image": "`+arvadostest.DockerImage112PDH+`",
1011     "cwd": "/bin",
1012     "environment": {},
1013     "mounts": {"/tmp": {"kind": "tmp"} },
1014     "output_path": "/tmp",
1015     "priority": 1,
1016     "runtime_constraints": {"cuda": {"device_count": 2}},
1017     "state": "Locked",
1018     "output_storage_classes": ["foo", "bar"]
1019 }`, nil, 0, func() {
1020                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1021         })
1022         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1023 }
1024
1025 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1026         s.fullRunHelper(c, `{
1027     "command": ["pwd"],
1028     "container_image": "`+arvadostest.DockerImage112PDH+`",
1029     "cwd": "/bin",
1030     "environment": {},
1031     "mounts": {"/tmp": {"kind": "tmp"} },
1032     "output_path": "/tmp",
1033     "priority": 1,
1034     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1035     "state": "Locked",
1036     "output_storage_classes": ["foo", "bar"]
1037 }`, nil, 0, func() {
1038                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1039         })
1040         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1041 }
1042
1043 func (s *TestSuite) TestStopOnSignal(c *C) {
1044         s.executor.runFunc = func() {
1045                 s.executor.created.Stdout.Write([]byte("foo\n"))
1046                 s.runner.SigChan <- syscall.SIGINT
1047         }
1048         s.testStopContainer(c)
1049 }
1050
1051 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1052         s.executor.runFunc = func() {
1053                 s.executor.created.Stdout.Write([]byte("foo\n"))
1054                 s.runner.ArvMountExit <- nil
1055                 close(s.runner.ArvMountExit)
1056         }
1057         s.runner.ArvMountExit = make(chan error)
1058         s.testStopContainer(c)
1059 }
1060
1061 func (s *TestSuite) testStopContainer(c *C) {
1062         record := `{
1063     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1064     "container_image": "` + arvadostest.DockerImage112PDH + `",
1065     "cwd": ".",
1066     "environment": {},
1067     "mounts": {"/tmp": {"kind": "tmp"} },
1068     "output_path": "/tmp",
1069     "priority": 1,
1070     "runtime_constraints": {},
1071     "state": "Locked"
1072 }`
1073
1074         err := json.Unmarshal([]byte(record), &s.api.Container)
1075         c.Assert(err, IsNil)
1076
1077         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1078         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1079                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1080         }
1081
1082         done := make(chan error)
1083         go func() {
1084                 done <- s.runner.Run()
1085         }()
1086         select {
1087         case <-time.After(20 * time.Second):
1088                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1089                 c.Fatal("timed out")
1090         case err = <-done:
1091                 c.Check(err, IsNil)
1092         }
1093         for k, v := range s.api.Logs {
1094                 c.Log(k)
1095                 c.Log(v.String(), "\n")
1096         }
1097
1098         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1099         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1100         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1101 }
1102
1103 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1104         s.fullRunHelper(c, `{
1105     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1106     "container_image": "`+arvadostest.DockerImage112PDH+`",
1107     "cwd": "/bin",
1108     "environment": {"FROBIZ": "bilbo"},
1109     "mounts": {"/tmp": {"kind": "tmp"} },
1110     "output_path": "/tmp",
1111     "priority": 1,
1112     "runtime_constraints": {},
1113     "state": "Locked"
1114 }`, nil, 0, func() {
1115                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1116         })
1117
1118         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1119         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1120         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1121 }
1122
1123 type ArvMountCmdLine struct {
1124         Cmd   []string
1125         token string
1126 }
1127
1128 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1129         am.Cmd = c
1130         am.token = token
1131         return nil, nil
1132 }
1133
1134 func stubCert(temp string) string {
1135         path := temp + "/ca-certificates.crt"
1136         crt, _ := os.Create(path)
1137         crt.Close()
1138         arvadosclient.CertFiles = []string{path}
1139         return path
1140 }
1141
1142 func (s *TestSuite) TestSetupMounts(c *C) {
1143         cr := s.runner
1144         am := &ArvMountCmdLine{}
1145         cr.RunArvMount = am.ArvMountTest
1146         cr.ContainerArvClient = &ArvTestClient{}
1147         cr.ContainerKeepClient = &KeepTestClient{}
1148         cr.Container.OutputStorageClasses = []string{"default"}
1149
1150         realTemp := c.MkDir()
1151         certTemp := c.MkDir()
1152         stubCertPath := stubCert(certTemp)
1153         cr.parentTemp = realTemp
1154
1155         i := 0
1156         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1157                 i++
1158                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1159                 err := os.Mkdir(d, os.ModePerm)
1160                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1161                         // Test case must have pre-populated the tempdir
1162                         err = nil
1163                 }
1164                 return d, err
1165         }
1166
1167         checkEmpty := func() {
1168                 // Should be deleted.
1169                 _, err := os.Stat(realTemp)
1170                 c.Assert(os.IsNotExist(err), Equals, true)
1171
1172                 // Now recreate it for the next test.
1173                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1174         }
1175
1176         {
1177                 i = 0
1178                 cr.ArvMountPoint = ""
1179                 cr.Container.Mounts = make(map[string]arvados.Mount)
1180                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1181                 cr.Container.OutputPath = "/tmp"
1182                 cr.statInterval = 5 * time.Second
1183                 bindmounts, err := cr.SetupMounts()
1184                 c.Check(err, IsNil)
1185                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1186                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1187                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1188                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1189                 os.RemoveAll(cr.ArvMountPoint)
1190                 cr.CleanupDirs()
1191                 checkEmpty()
1192         }
1193
1194         {
1195                 i = 0
1196                 cr.ArvMountPoint = ""
1197                 cr.Container.Mounts = make(map[string]arvados.Mount)
1198                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1199                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1200                 cr.Container.OutputPath = "/out"
1201                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1202
1203                 bindmounts, err := cr.SetupMounts()
1204                 c.Check(err, IsNil)
1205                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1206                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1207                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1208                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1209                 os.RemoveAll(cr.ArvMountPoint)
1210                 cr.CleanupDirs()
1211                 checkEmpty()
1212         }
1213
1214         {
1215                 i = 0
1216                 cr.ArvMountPoint = ""
1217                 cr.Container.Mounts = make(map[string]arvados.Mount)
1218                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1219                 cr.Container.OutputPath = "/tmp"
1220                 cr.Container.RuntimeConstraints.API = true
1221                 cr.Container.OutputStorageClasses = []string{"default"}
1222
1223                 bindmounts, err := cr.SetupMounts()
1224                 c.Check(err, IsNil)
1225                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1226                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1227                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1228                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1229                 os.RemoveAll(cr.ArvMountPoint)
1230                 cr.CleanupDirs()
1231                 checkEmpty()
1232
1233                 cr.Container.RuntimeConstraints.API = false
1234         }
1235
1236         {
1237                 i = 0
1238                 cr.ArvMountPoint = ""
1239                 cr.Container.Mounts = map[string]arvados.Mount{
1240                         "/keeptmp": {Kind: "collection", Writable: true},
1241                 }
1242                 cr.Container.OutputPath = "/keeptmp"
1243
1244                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1245
1246                 bindmounts, err := cr.SetupMounts()
1247                 c.Check(err, IsNil)
1248                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1249                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1250                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1251                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1252                 os.RemoveAll(cr.ArvMountPoint)
1253                 cr.CleanupDirs()
1254                 checkEmpty()
1255         }
1256
1257         {
1258                 i = 0
1259                 cr.ArvMountPoint = ""
1260                 cr.Container.Mounts = map[string]arvados.Mount{
1261                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1262                         "/keepout": {Kind: "collection", Writable: true},
1263                 }
1264                 cr.Container.OutputPath = "/keepout"
1265
1266                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1267                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1268
1269                 bindmounts, err := cr.SetupMounts()
1270                 c.Check(err, IsNil)
1271                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1272                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1273                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1274                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1275                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1276                         "/keepout": {realTemp + "/keep1/tmp0", false},
1277                 })
1278                 os.RemoveAll(cr.ArvMountPoint)
1279                 cr.CleanupDirs()
1280                 checkEmpty()
1281         }
1282
1283         {
1284                 i = 0
1285                 cr.ArvMountPoint = ""
1286                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1287                 cr.Container.Mounts = map[string]arvados.Mount{
1288                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1289                         "/keepout": {Kind: "collection", Writable: true},
1290                 }
1291                 cr.Container.OutputPath = "/keepout"
1292
1293                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1294                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1295
1296                 bindmounts, err := cr.SetupMounts()
1297                 c.Check(err, IsNil)
1298                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1299                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1300                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1301                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1302                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1303                         "/keepout": {realTemp + "/keep1/tmp0", false},
1304                 })
1305                 os.RemoveAll(cr.ArvMountPoint)
1306                 cr.CleanupDirs()
1307                 checkEmpty()
1308         }
1309
1310         for _, test := range []struct {
1311                 in  interface{}
1312                 out string
1313         }{
1314                 {in: "foo", out: `"foo"`},
1315                 {in: nil, out: `null`},
1316                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1317         } {
1318                 i = 0
1319                 cr.ArvMountPoint = ""
1320                 cr.Container.Mounts = map[string]arvados.Mount{
1321                         "/mnt/test.json": {Kind: "json", Content: test.in},
1322                 }
1323                 bindmounts, err := cr.SetupMounts()
1324                 c.Check(err, IsNil)
1325                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1326                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1327                 })
1328                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1329                 c.Check(err, IsNil)
1330                 c.Check(content, DeepEquals, []byte(test.out))
1331                 os.RemoveAll(cr.ArvMountPoint)
1332                 cr.CleanupDirs()
1333                 checkEmpty()
1334         }
1335
1336         for _, test := range []struct {
1337                 in  interface{}
1338                 out string
1339         }{
1340                 {in: "foo", out: `foo`},
1341                 {in: nil, out: "error"},
1342                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1343         } {
1344                 i = 0
1345                 cr.ArvMountPoint = ""
1346                 cr.Container.Mounts = map[string]arvados.Mount{
1347                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1348                 }
1349                 bindmounts, err := cr.SetupMounts()
1350                 if test.out == "error" {
1351                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1352                 } else {
1353                         c.Check(err, IsNil)
1354                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1355                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1356                         })
1357                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1358                         c.Check(err, IsNil)
1359                         c.Check(content, DeepEquals, []byte(test.out))
1360                 }
1361                 os.RemoveAll(cr.ArvMountPoint)
1362                 cr.CleanupDirs()
1363                 checkEmpty()
1364         }
1365
1366         // Read-only mount points are allowed underneath output_dir mount point
1367         {
1368                 i = 0
1369                 cr.ArvMountPoint = ""
1370                 cr.Container.Mounts = make(map[string]arvados.Mount)
1371                 cr.Container.Mounts = map[string]arvados.Mount{
1372                         "/tmp":     {Kind: "tmp"},
1373                         "/tmp/foo": {Kind: "collection"},
1374                 }
1375                 cr.Container.OutputPath = "/tmp"
1376
1377                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1378
1379                 bindmounts, err := cr.SetupMounts()
1380                 c.Check(err, IsNil)
1381                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1382                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1383                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1384                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1385                         "/tmp":     {realTemp + "/tmp2", false},
1386                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1387                 })
1388                 os.RemoveAll(cr.ArvMountPoint)
1389                 cr.CleanupDirs()
1390                 checkEmpty()
1391         }
1392
1393         // Writable mount points copied to output_dir mount point
1394         {
1395                 i = 0
1396                 cr.ArvMountPoint = ""
1397                 cr.Container.Mounts = make(map[string]arvados.Mount)
1398                 cr.Container.Mounts = map[string]arvados.Mount{
1399                         "/tmp": {Kind: "tmp"},
1400                         "/tmp/foo": {Kind: "collection",
1401                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1402                                 Writable:         true},
1403                         "/tmp/bar": {Kind: "collection",
1404                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1405                                 Path:             "baz",
1406                                 Writable:         true},
1407                 }
1408                 cr.Container.OutputPath = "/tmp"
1409
1410                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1411                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1412
1413                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1414                 rf.Write([]byte("bar"))
1415                 rf.Close()
1416
1417                 _, err := cr.SetupMounts()
1418                 c.Check(err, IsNil)
1419                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1420                 c.Check(err, IsNil)
1421                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1422                 c.Check(err, IsNil)
1423                 os.RemoveAll(cr.ArvMountPoint)
1424                 cr.CleanupDirs()
1425                 checkEmpty()
1426         }
1427
1428         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1429         {
1430                 i = 0
1431                 cr.ArvMountPoint = ""
1432                 cr.Container.Mounts = make(map[string]arvados.Mount)
1433                 cr.Container.Mounts = map[string]arvados.Mount{
1434                         "/tmp":     {Kind: "tmp"},
1435                         "/tmp/foo": {Kind: "tmp"},
1436                 }
1437                 cr.Container.OutputPath = "/tmp"
1438
1439                 _, err := cr.SetupMounts()
1440                 c.Check(err, NotNil)
1441                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1442                 os.RemoveAll(cr.ArvMountPoint)
1443                 cr.CleanupDirs()
1444                 checkEmpty()
1445         }
1446
1447         // Only mount point of kind 'collection' is allowed for stdin
1448         {
1449                 i = 0
1450                 cr.ArvMountPoint = ""
1451                 cr.Container.Mounts = make(map[string]arvados.Mount)
1452                 cr.Container.Mounts = map[string]arvados.Mount{
1453                         "stdin": {Kind: "tmp"},
1454                 }
1455
1456                 _, err := cr.SetupMounts()
1457                 c.Check(err, NotNil)
1458                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1459                 os.RemoveAll(cr.ArvMountPoint)
1460                 cr.CleanupDirs()
1461                 checkEmpty()
1462         }
1463
1464         // git_tree mounts
1465         {
1466                 i = 0
1467                 cr.ArvMountPoint = ""
1468                 (*GitMountSuite)(nil).useTestGitServer(c)
1469                 cr.token = arvadostest.ActiveToken
1470                 cr.Container.Mounts = make(map[string]arvados.Mount)
1471                 cr.Container.Mounts = map[string]arvados.Mount{
1472                         "/tip": {
1473                                 Kind:   "git_tree",
1474                                 UUID:   arvadostest.Repository2UUID,
1475                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1476                                 Path:   "/",
1477                         },
1478                         "/non-tip": {
1479                                 Kind:   "git_tree",
1480                                 UUID:   arvadostest.Repository2UUID,
1481                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1482                                 Path:   "/",
1483                         },
1484                 }
1485                 cr.Container.OutputPath = "/tmp"
1486
1487                 bindmounts, err := cr.SetupMounts()
1488                 c.Check(err, IsNil)
1489
1490                 for path, mount := range bindmounts {
1491                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1492                 }
1493
1494                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1495                 c.Check(err, IsNil)
1496                 c.Check(string(data), Equals, "\000\001\002\003")
1497                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1498                 c.Check(err, FitsTypeOf, &os.PathError{})
1499                 c.Check(os.IsNotExist(err), Equals, true)
1500
1501                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1502                 c.Check(err, IsNil)
1503                 c.Check(string(data), Equals, "\000\001\002\003")
1504                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1505                 c.Check(err, IsNil)
1506                 c.Check(string(data), Equals, "testfile\n")
1507
1508                 cr.CleanupDirs()
1509                 checkEmpty()
1510         }
1511 }
1512
1513 func (s *TestSuite) TestStdout(c *C) {
1514         helperRecord := `{
1515                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1516                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1517                 "cwd": "/bin",
1518                 "environment": {"FROBIZ": "bilbo"},
1519                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1520                 "output_path": "/tmp",
1521                 "priority": 1,
1522                 "runtime_constraints": {},
1523                 "state": "Locked"
1524         }`
1525
1526         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1527                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1528         })
1529
1530         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1531         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1532         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1533 }
1534
1535 // Used by the TestStdoutWithWrongPath*()
1536 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1537         err := json.Unmarshal([]byte(record), &s.api.Container)
1538         c.Assert(err, IsNil)
1539         s.executor.runFunc = fn
1540         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1541         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1542                 return s.api, &KeepTestClient{}, nil, nil
1543         }
1544         return s.api, s.runner, s.runner.Run()
1545 }
1546
1547 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1548         _, _, err := s.stdoutErrorRunHelper(c, `{
1549     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1550     "output_path": "/tmp",
1551     "state": "Locked"
1552 }`, func() {})
1553         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1554 }
1555
1556 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1557         _, _, err := s.stdoutErrorRunHelper(c, `{
1558     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1559     "output_path": "/tmp",
1560     "state": "Locked"
1561 }`, func() {})
1562         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1563 }
1564
1565 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1566         _, _, err := s.stdoutErrorRunHelper(c, `{
1567     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1568     "output_path": "/tmp",
1569     "state": "Locked"
1570 }`, func() {})
1571         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1572 }
1573
1574 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1575         s.fullRunHelper(c, `{
1576     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1577     "container_image": "`+arvadostest.DockerImage112PDH+`",
1578     "cwd": "/bin",
1579     "environment": {},
1580     "mounts": {"/tmp": {"kind": "tmp"} },
1581     "output_path": "/tmp",
1582     "priority": 1,
1583     "runtime_constraints": {"API": true},
1584     "state": "Locked"
1585 }`, nil, 0, func() {
1586                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1587                 s.executor.exit <- 3
1588         })
1589         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1590         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1591         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1592 }
1593
1594 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1595         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1596         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1597         s.fullRunHelper(c, `{
1598     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1599     "container_image": "`+arvadostest.DockerImage112PDH+`",
1600     "cwd": "/bin",
1601     "environment": {},
1602     "mounts": {"/tmp": {"kind": "tmp"} },
1603     "output_path": "/tmp",
1604     "priority": 1,
1605     "runtime_constraints": {"API": true},
1606     "state": "Locked"
1607 }`, nil, 0, func() {
1608                 s.api.Container.Output = arvadostest.DockerImage112PDH
1609         })
1610
1611         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1612         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1613         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1614 }
1615
1616 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1617         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1618                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1619                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1620                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1621         }
1622         s.executor.runFunc = func() {
1623                 time.Sleep(time.Second)
1624                 s.executor.exit <- 137
1625         }
1626         record := `{
1627     "command": ["sleep", "1"],
1628     "container_image": "` + arvadostest.DockerImage112PDH + `",
1629     "cwd": "/bin",
1630     "environment": {},
1631     "mounts": {"/tmp": {"kind": "tmp"} },
1632     "output_path": "/tmp",
1633     "priority": 1,
1634     "runtime_constraints": {"API": true},
1635     "state": "Locked"
1636 }`
1637         err := json.Unmarshal([]byte(record), &s.api.Container)
1638         c.Assert(err, IsNil)
1639         err = s.runner.Run()
1640         c.Assert(err, IsNil)
1641         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1642         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1643         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1644         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1645         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1646 }
1647
1648 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1649         helperRecord := `{
1650                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1651                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1652                 "cwd": "/bin",
1653                 "environment": {"FROBIZ": "bilbo"},
1654                 "mounts": {
1655         "/tmp": {"kind": "tmp"},
1656         "/tmp/foo": {"kind": "collection",
1657                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1658                      "exclude_from_output": true
1659         },
1660         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1661     },
1662                 "output_path": "/tmp",
1663                 "priority": 1,
1664                 "runtime_constraints": {},
1665                 "state": "Locked"
1666         }`
1667
1668         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1669
1670         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1671                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1672         })
1673
1674         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1675         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1676         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1677 }
1678
1679 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1680         helperRecord := `{
1681                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1682                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1683                 "cwd": "/bin",
1684                 "environment": {"FROBIZ": "bilbo"},
1685                 "mounts": {
1686         "/tmp": {"kind": "tmp"},
1687         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1688         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1689         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1690         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1691         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1692     },
1693                 "output_path": "/tmp",
1694                 "priority": 1,
1695                 "runtime_constraints": {},
1696                 "state": "Locked"
1697         }`
1698
1699         extraMounts := []string{
1700                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1701                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1702                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1703         }
1704
1705         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1706                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1707         })
1708
1709         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1710                 "/tmp":                   {realtemp + "/tmp1", false},
1711                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1712                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1713                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1714                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1715         })
1716
1717         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1718         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1719         for _, v := range api.Content {
1720                 if v["collection"] != nil {
1721                         c.Check(v["ensure_unique_name"], Equals, true)
1722                         collection := v["collection"].(arvadosclient.Dict)
1723                         if strings.Index(collection["name"].(string), "output") == 0 {
1724                                 manifest := collection["manifest_text"].(string)
1725
1726                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1727 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1728 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1729 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1730 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1731 `)
1732                         }
1733                 }
1734         }
1735 }
1736
1737 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1738         helperRecord := `{
1739                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1740                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1741                 "cwd": "/bin",
1742                 "environment": {"FROBIZ": "bilbo"},
1743                 "mounts": {
1744         "/tmp": {"kind": "tmp"},
1745         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1746         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1747     },
1748                 "output_path": "/tmp",
1749                 "priority": 1,
1750                 "runtime_constraints": {},
1751                 "state": "Locked"
1752         }`
1753
1754         extraMounts := []string{
1755                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1756         }
1757
1758         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1759                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1760         })
1761
1762         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1763         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1764         for _, v := range s.api.Content {
1765                 if v["collection"] != nil {
1766                         collection := v["collection"].(arvadosclient.Dict)
1767                         if strings.Index(collection["name"].(string), "output") == 0 {
1768                                 manifest := collection["manifest_text"].(string)
1769
1770                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1771 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1772 `)
1773                         }
1774                 }
1775         }
1776 }
1777
1778 func (s *TestSuite) TestOutputError(c *C) {
1779         helperRecord := `{
1780                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1781                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1782                 "cwd": "/bin",
1783                 "environment": {"FROBIZ": "bilbo"},
1784                 "mounts": {
1785                         "/tmp": {"kind": "tmp"}
1786                 },
1787                 "output_path": "/tmp",
1788                 "priority": 1,
1789                 "runtime_constraints": {},
1790                 "state": "Locked"
1791         }`
1792         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1793                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1794         })
1795
1796         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1797 }
1798
1799 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1800         helperRecord := `{
1801                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1802                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1803                 "cwd": "/bin",
1804                 "environment": {"FROBIZ": "bilbo"},
1805                 "mounts": {
1806         "/tmp": {"kind": "tmp"},
1807         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1808         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1809     },
1810                 "output_path": "/tmp",
1811                 "priority": 1,
1812                 "runtime_constraints": {},
1813                 "state": "Locked"
1814         }`
1815
1816         extraMounts := []string{
1817                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1818         }
1819
1820         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1821                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1822         })
1823
1824         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1825         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1826         for _, v := range api.Content {
1827                 if v["collection"] != nil {
1828                         collection := v["collection"].(arvadosclient.Dict)
1829                         if strings.Index(collection["name"].(string), "output") == 0 {
1830                                 manifest := collection["manifest_text"].(string)
1831                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1832 `)
1833                         }
1834                 }
1835         }
1836 }
1837
1838 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1839         helperRecord := `{
1840                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1841                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1842                 "cwd": "/bin",
1843                 "environment": {"FROBIZ": "bilbo"},
1844                 "mounts": {
1845         "/tmp": {"kind": "tmp"},
1846         "stdin": {"kind": "json", "content": "foo"},
1847         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1848     },
1849                 "output_path": "/tmp",
1850                 "priority": 1,
1851                 "runtime_constraints": {},
1852                 "state": "Locked"
1853         }`
1854
1855         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1856                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1857         })
1858
1859         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1860         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1861         for _, v := range api.Content {
1862                 if v["collection"] != nil {
1863                         collection := v["collection"].(arvadosclient.Dict)
1864                         if strings.Index(collection["name"].(string), "output") == 0 {
1865                                 manifest := collection["manifest_text"].(string)
1866                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1867 `)
1868                         }
1869                 }
1870         }
1871 }
1872
1873 func (s *TestSuite) TestStderrMount(c *C) {
1874         api, cr, _ := s.fullRunHelper(c, `{
1875     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1876     "container_image": "`+arvadostest.DockerImage112PDH+`",
1877     "cwd": ".",
1878     "environment": {},
1879     "mounts": {"/tmp": {"kind": "tmp"},
1880                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1881                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1882     "output_path": "/tmp",
1883     "priority": 1,
1884     "runtime_constraints": {},
1885     "state": "Locked"
1886 }`, nil, 1, func() {
1887                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1888                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1889         })
1890
1891         final := api.CalledWith("container.state", "Complete")
1892         c.Assert(final, NotNil)
1893         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1894         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1895
1896         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1897 }
1898
1899 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1900         s.api.callraw = true
1901         err := s.runner.fetchContainerRecord()
1902         c.Assert(err, IsNil)
1903         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1904         c.Logf("%#v", s.runner.Container)
1905         c.Check(err, IsNil)
1906         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1907 }
1908
1909 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1910         nextState := ""
1911         for _, setup := range []func(){
1912                 func() {
1913                         c.Log("// waitErr = ocl runtime error")
1914                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1915                         nextState = "Cancelled"
1916                 },
1917                 func() {
1918                         c.Log("// loadErr = cannot connect")
1919                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1920                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1921                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1922                         c.Assert(err, IsNil)
1923                         nextState = "Queued"
1924                 },
1925         } {
1926                 s.SetUpTest(c)
1927                 setup()
1928                 s.fullRunHelper(c, `{
1929     "command": ["echo", "hello world"],
1930     "container_image": "`+arvadostest.DockerImage112PDH+`",
1931     "cwd": ".",
1932     "environment": {},
1933     "mounts": {"/tmp": {"kind": "tmp"} },
1934     "output_path": "/tmp",
1935     "priority": 1,
1936     "runtime_constraints": {},
1937     "state": "Locked"
1938 }`, nil, 0, func() {})
1939                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1940                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1941                 if s.runner.brokenNodeHook != "" {
1942                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1943                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1944                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1945                 } else {
1946                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1947                 }
1948         }
1949 }
1950
1951 func (s *TestSuite) TestBadCommand(c *C) {
1952         for _, startError := range []string{
1953                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1954                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1955                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1956         } {
1957                 s.SetUpTest(c)
1958                 s.executor.startErr = errors.New(startError)
1959                 s.fullRunHelper(c, `{
1960     "command": ["echo", "hello world"],
1961     "container_image": "`+arvadostest.DockerImage112PDH+`",
1962     "cwd": ".",
1963     "environment": {},
1964     "mounts": {"/tmp": {"kind": "tmp"} },
1965     "output_path": "/tmp",
1966     "priority": 1,
1967     "runtime_constraints": {},
1968     "state": "Locked"
1969 }`, nil, 0, func() {})
1970                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1971                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1972         }
1973 }
1974
1975 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1976         helperRecord := `{
1977                 "command": ["true"],
1978                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1979                 "cwd": "/bin",
1980                 "mounts": {
1981                     "/tmp": {"kind": "tmp"},
1982                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1983                 },
1984                 "secret_mounts": {
1985                 },
1986                 "output_path": "/tmp",
1987                 "priority": 1,
1988                 "runtime_constraints": {},
1989                 "state": "Locked"
1990         }`
1991
1992         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1993                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1994                 c.Check(err, IsNil)
1995                 c.Check(string(content), Equals, "mypassword")
1996         })
1997
1998         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1999         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2000         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
2001         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
2002
2003         // under secret mounts, not captured in output
2004         helperRecord = `{
2005                 "command": ["true"],
2006                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2007                 "cwd": "/bin",
2008                 "mounts": {
2009                     "/tmp": {"kind": "tmp"}
2010                 },
2011                 "secret_mounts": {
2012                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2013                 },
2014                 "output_path": "/tmp",
2015                 "priority": 1,
2016                 "runtime_constraints": {},
2017                 "state": "Locked"
2018         }`
2019
2020         s.SetUpTest(c)
2021         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2022                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2023                 c.Check(err, IsNil)
2024                 c.Check(string(content), Equals, "mypassword")
2025         })
2026
2027         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2028         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2029         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2030         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2031
2032         // under secret mounts, output dir is a collection, not captured in output
2033         helperRecord = `{
2034                 "command": ["true"],
2035                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2036                 "cwd": "/bin",
2037                 "mounts": {
2038                     "/tmp": {"kind": "collection", "writable": true}
2039                 },
2040                 "secret_mounts": {
2041                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2042                 },
2043                 "output_path": "/tmp",
2044                 "priority": 1,
2045                 "runtime_constraints": {},
2046                 "state": "Locked"
2047         }`
2048
2049         s.SetUpTest(c)
2050         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2051                 // secret.conf should be provisioned as a separate
2052                 // bind mount, i.e., it should not appear in the
2053                 // (fake) fuse filesystem as viewed from the host.
2054                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2055                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2056                         c.Logf("secret.conf: content %q, err %#v", content, err)
2057                 }
2058                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2059                 c.Check(err, IsNil)
2060         })
2061
2062         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2063         c.Check(err, IsNil)
2064         c.Check(string(content), Equals, "mypassword")
2065         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2066         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2067         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2068         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2069 }
2070
2071 type FakeProcess struct {
2072         cmdLine []string
2073 }
2074
2075 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2076         return fp.cmdLine, nil
2077 }