lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         keepmountTmp             []string
  48         testDispatcherKeepClient KeepTestClient
  49         testContainerKeepClient  KeepTestClient
  50 }
  51
  52 func (s *TestSuite) SetUpTest(c *C) {
  53         *brokenNodeHook = ""
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 133 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 134 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 135 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 136 func (e *stubExecutor) Close()                          { e.closed = true }
 137 func (e *stubExecutor) Wait(context.Context) (int, error) {
 138         return <-e.exit, e.waitErr
 139 }
 140
 141 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 142
 143 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 144 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 145 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 146
 147 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 148 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 149
 150 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 151 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 152 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 153 `
 154
 155 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 156
 157 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 158 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 159
 160 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 161 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 162
 163 func (client *ArvTestClient) Create(resourceType string,
 164         parameters arvadosclient.Dict,
 165         output interface{}) error {
 166
 167         client.Mutex.Lock()
 168         defer client.Mutex.Unlock()
 169
 170         client.Calls++
 171         client.Content = append(client.Content, parameters)
 172
 173         if resourceType == "logs" {
 174                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 175                 if client.Logs == nil {
 176                         client.Logs = make(map[string]*bytes.Buffer)
 177                 }
 178                 if client.Logs[et] == nil {
 179                         client.Logs[et] = &bytes.Buffer{}
 180                 }
 181                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 182         }
 183
 184         if resourceType == "collections" && output != nil {
 185                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 186                 outmap := output.(*arvados.Collection)
 187                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 188                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 189         }
 190
 191         return nil
 192 }
 193
 194 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 195         switch {
 196         case method == "GET" && resourceType == "containers" && action == "auth":
 197                 return json.Unmarshal([]byte(`{
 198                         "kind": "arvados#api_client_authorization",
 199                         "uuid": "`+fakeAuthUUID+`",
 200                         "api_token": "`+fakeAuthToken+`"
 201                         }`), output)
 202         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 203                 if client.secretMounts != nil {
 204                         return json.Unmarshal(client.secretMounts, output)
 205                 }
 206                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 207         default:
 208                 return fmt.Errorf("Not found")
 209         }
 210 }
 211
 212 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 213         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 214         var j []byte
 215         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 216                 j = []byte(`{
 217                         "kind": "arvados#nodeList",
 218                         "items": [{
 219                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 220                                 "hostname": "compute2",
 221                                 "properties": {"total_cpu_cores": 16}
 222                         }]}`)
 223         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 224                 if uuid == "" {
 225                         j, err = json.Marshal(map[string]interface{}{
 226                                 "items": []interface{}{client.Container},
 227                                 "kind":  "arvados#nodeList",
 228                         })
 229                 } else {
 230                         j, err = json.Marshal(client.Container)
 231                 }
 232         } else {
 233                 j = []byte(`{
 234                         "command": ["sleep", "1"],
 235                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 236                         "cwd": ".",
 237                         "environment": {},
 238                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 239                         "output_path": "/tmp",
 240                         "priority": 1,
 241                         "runtime_constraints": {}
 242                 }`)
 243         }
 244         return ioutil.NopCloser(bytes.NewReader(j)), err
 245 }
 246
 247 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 248         if resourceType == "collections" {
 249                 if uuid == hwPDH {
 250                         output.(*arvados.Collection).ManifestText = hwManifest
 251                 } else if uuid == otherPDH {
 252                         output.(*arvados.Collection).ManifestText = otherManifest
 253                 } else if uuid == normalizedWithSubdirsPDH {
 254                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 255                 } else if uuid == denormalizedWithSubdirsPDH {
 256                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 257                 }
 258         }
 259         if resourceType == "containers" {
 260                 (*output.(*arvados.Container)) = client.Container
 261         }
 262         return nil
 263 }
 264
 265 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 266         client.Mutex.Lock()
 267         defer client.Mutex.Unlock()
 268         client.Calls++
 269         client.Content = append(client.Content, parameters)
 270         if resourceType == "containers" {
 271                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 272                         client.WasSetRunning = true
 273                 }
 274         } else if resourceType == "collections" {
 275                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 276                 output.(*arvados.Collection).UUID = uuid
 277                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 278         }
 279         return nil
 280 }
 281
 282 var discoveryMap = map[string]interface{}{
 283         "defaultTrashLifetime":               float64(1209600),
 284         "crunchLimitLogBytesPerJob":          float64(67108864),
 285         "crunchLogThrottleBytes":             float64(65536),
 286         "crunchLogThrottlePeriod":            float64(60),
 287         "crunchLogThrottleLines":             float64(1024),
 288         "crunchLogPartialLineThrottlePeriod": float64(5),
 289         "crunchLogBytesPerEvent":             float64(4096),
 290         "crunchLogSecondsBetweenEvents":      float64(1),
 291 }
 292
 293 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 294         return discoveryMap[key], nil
 295 }
 296
 297 // CalledWith returns the parameters from the first API call whose
 298 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 299 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 300 // no call matches, it returns nil.
 301 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 302 call:
 303         for _, content := range client.Content {
 304                 var v interface{} = content
 305                 for _, k := range strings.Split(jpath, ".") {
 306                         if dict, ok := v.(arvadosclient.Dict); !ok {
 307                                 continue call
 308                         } else {
 309                                 v = dict[k]
 310                         }
 311                 }
 312                 if v == expect {
 313                         return content
 314                 }
 315         }
 316         return nil
 317 }
 318
 319 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 320         return locator, nil
 321 }
 322
 323 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 324         client.Content = opts.Data
 325         return arvados.BlockWriteResponse{
 326                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 327         }, nil
 328 }
 329
 330 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 331         return 0, errors.New("not implemented")
 332 }
 333
 334 func (client *KeepTestClient) ClearBlockCache() {
 335 }
 336
 337 func (client *KeepTestClient) Close() {
 338         client.Content = nil
 339 }
 340
 341 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 342         client.StorageClasses = sc
 343 }
 344
 345 type FileWrapper struct {
 346         io.ReadCloser
 347         len int64
 348 }
 349
 350 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 351         return nil, errors.New("not implemented")
 352 }
 353
 354 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 355         return 0, errors.New("not implemented")
 356 }
 357
 358 func (fw FileWrapper) Size() int64 {
 359         return fw.len
 360 }
 361
 362 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 363         return nil, errors.New("not implemented")
 364 }
 365
 366 func (fw FileWrapper) Truncate(int64) error {
 367         return errors.New("not implemented")
 368 }
 369
 370 func (fw FileWrapper) Write([]byte) (int, error) {
 371         return 0, errors.New("not implemented")
 372 }
 373
 374 func (fw FileWrapper) Sync() error {
 375         return errors.New("not implemented")
 376 }
 377
 378 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 379         if filename == hwImageID+".tar" {
 380                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 381                 client.Called = true
 382                 return FileWrapper{rdr, 1321984}, nil
 383         } else if filename == "/file1_in_main.txt" {
 384                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 385                 client.Called = true
 386                 return FileWrapper{rdr, 3}, nil
 387         }
 388         return nil, nil
 389 }
 390
 391 func (s *TestSuite) TestLoadImage(c *C) {
 392         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 393         s.runner.Container.Mounts = map[string]arvados.Mount{
 394                 "/out": {Kind: "tmp", Writable: true},
 395         }
 396         s.runner.Container.OutputPath = "/out"
 397
 398         _, err := s.runner.SetupMounts()
 399         c.Assert(err, IsNil)
 400
 401         imageID, err := s.runner.LoadImage()
 402         c.Check(err, IsNil)
 403         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 404         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 405
 406         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 407         s.executor.imageLoaded = false
 408         s.executor.loaded = ""
 409         s.executor.loadErr = errors.New("bork")
 410         imageID, err = s.runner.LoadImage()
 411         c.Check(err, ErrorMatches, ".*bork")
 412         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 413
 414         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 415         s.executor.imageLoaded = false
 416         s.executor.loaded = ""
 417         s.executor.loadErr = nil
 418         imageID, err = s.runner.LoadImage()
 419         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 420         c.Check(s.executor.loaded, Equals, "")
 421 }
 422
 423 type ArvErrorTestClient struct{}
 424
 425 func (ArvErrorTestClient) Create(resourceType string,
 426         parameters arvadosclient.Dict,
 427         output interface{}) error {
 428         return nil
 429 }
 430
 431 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 432         if method == "GET" && resourceType == "containers" && action == "auth" {
 433                 return nil
 434         }
 435         return errors.New("ArvError")
 436 }
 437
 438 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 439         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 440         return nil, errors.New("ArvError")
 441 }
 442
 443 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 444         return errors.New("ArvError")
 445 }
 446
 447 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 448         return nil
 449 }
 450
 451 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 452         return discoveryMap[key], nil
 453 }
 454
 455 type KeepErrorTestClient struct {
 456         KeepTestClient
 457 }
 458
 459 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 460         return nil, errors.New("KeepError")
 461 }
 462
 463 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 464         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 465 }
 466
 467 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 468         return "", errors.New("KeepError")
 469 }
 470
 471 type KeepReadErrorTestClient struct {
 472         KeepTestClient
 473 }
 474
 475 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 476         return 0, errors.New("KeepError")
 477 }
 478
 479 type ErrorReader struct {
 480         FileWrapper
 481 }
 482
 483 func (ErrorReader) Read(p []byte) (n int, err error) {
 484         return 0, errors.New("ErrorReader")
 485 }
 486
 487 func (ErrorReader) Seek(int64, int) (int64, error) {
 488         return 0, errors.New("ErrorReader")
 489 }
 490
 491 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 492         return ErrorReader{}, nil
 493 }
 494
 495 type ClosableBuffer struct {
 496         bytes.Buffer
 497 }
 498
 499 func (*ClosableBuffer) Close() error {
 500         return nil
 501 }
 502
 503 type TestLogs struct {
 504         Stdout ClosableBuffer
 505         Stderr ClosableBuffer
 506 }
 507
 508 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 509         if logstr == "stdout" {
 510                 return &tl.Stdout, nil
 511         }
 512         if logstr == "stderr" {
 513                 return &tl.Stderr, nil
 514         }
 515         return nil, errors.New("???")
 516 }
 517
 518 func dockerLog(fd byte, msg string) []byte {
 519         by := []byte(msg)
 520         header := make([]byte, 8+len(by))
 521         header[0] = fd
 522         header[7] = byte(len(by))
 523         copy(header[8:], by)
 524         return header
 525 }
 526
 527 func (s *TestSuite) TestRunContainer(c *C) {
 528         s.executor.runFunc = func() {
 529                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 530                 s.executor.exit <- 0
 531         }
 532
 533         var logs TestLogs
 534         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 535         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 536         s.runner.Container.Command = []string{"./hw"}
 537         s.runner.Container.OutputStorageClasses = []string{"default"}
 538
 539         imageID, err := s.runner.LoadImage()
 540         c.Assert(err, IsNil)
 541
 542         err = s.runner.CreateContainer(imageID, nil)
 543         c.Assert(err, IsNil)
 544
 545         err = s.runner.StartContainer()
 546         c.Assert(err, IsNil)
 547
 548         err = s.runner.WaitFinish()
 549         c.Assert(err, IsNil)
 550
 551         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 552         c.Check(logs.Stderr.String(), Equals, "")
 553 }
 554
 555 func (s *TestSuite) TestCommitLogs(c *C) {
 556         api := &ArvTestClient{}
 557         kc := &KeepTestClient{}
 558         defer kc.Close()
 559         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 560         c.Assert(err, IsNil)
 561         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 562
 563         cr.CrunchLog.Print("Hello world!")
 564         cr.CrunchLog.Print("Goodbye")
 565         cr.finalState = "Complete"
 566
 567         err = cr.CommitLogs()
 568         c.Check(err, IsNil)
 569
 570         c.Check(api.Calls, Equals, 2)
 571         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 572         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 573         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 574         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 575 }
 576
 577 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 578         api := &ArvTestClient{}
 579         kc := &KeepTestClient{}
 580         defer kc.Close()
 581         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 582         c.Assert(err, IsNil)
 583
 584         err = cr.UpdateContainerRunning()
 585         c.Check(err, IsNil)
 586
 587         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 588 }
 589
 590 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 591         api := &ArvTestClient{}
 592         kc := &KeepTestClient{}
 593         defer kc.Close()
 594         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 595         c.Assert(err, IsNil)
 596
 597         cr.LogsPDH = new(string)
 598         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 599
 600         cr.ExitCode = new(int)
 601         *cr.ExitCode = 42
 602         cr.finalState = "Complete"
 603
 604         err = cr.UpdateContainerFinal()
 605         c.Check(err, IsNil)
 606
 607         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 608         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 609         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 610 }
 611
 612 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 613         api := &ArvTestClient{}
 614         kc := &KeepTestClient{}
 615         defer kc.Close()
 616         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 617         c.Assert(err, IsNil)
 618         cr.cCancelled = true
 619         cr.finalState = "Cancelled"
 620
 621         err = cr.UpdateContainerFinal()
 622         c.Check(err, IsNil)
 623
 624         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 625         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 626         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 627 }
 628
 629 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 630 // dress rehearsal of the Run() function, starting from a JSON container record.
 631 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 632         err := json.Unmarshal([]byte(record), &s.api.Container)
 633         c.Assert(err, IsNil)
 634         initialState := s.api.Container.State
 635
 636         var sm struct {
 637                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 638         }
 639         err = json.Unmarshal([]byte(record), &sm)
 640         c.Check(err, IsNil)
 641         secretMounts, err := json.Marshal(sm)
 642         c.Assert(err, IsNil)
 643         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 644
 645         s.executor.runFunc = func() {
 646                 fn()
 647                 s.executor.exit <- exitCode
 648         }
 649
 650         s.runner.statInterval = 100 * time.Millisecond
 651         s.runner.containerWatchdogInterval = time.Second
 652
 653         realTemp := c.MkDir()
 654         tempcount := 0
 655         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 656                 tempcount++
 657                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 658                 err := os.Mkdir(d, os.ModePerm)
 659                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 660                         // Test case must have pre-populated the tempdir
 661                         err = nil
 662                 }
 663                 return d, err
 664         }
 665         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 666                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 667         }
 668
 669         if extraMounts != nil && len(extraMounts) > 0 {
 670                 err := s.runner.SetupArvMountPoint("keep")
 671                 c.Check(err, IsNil)
 672
 673                 for _, m := range extraMounts {
 674                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 675                 }
 676         }
 677
 678         err = s.runner.Run()
 679         if s.api.CalledWith("container.state", "Complete") != nil {
 680                 c.Check(err, IsNil)
 681         }
 682         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 683                 c.Check(s.api.WasSetRunning, Equals, true)
 684                 var lastupdate arvadosclient.Dict
 685                 for _, content := range s.api.Content {
 686                         if content["container"] != nil {
 687                                 lastupdate = content["container"].(arvadosclient.Dict)
 688                         }
 689                 }
 690                 if lastupdate["log"] == nil {
 691                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 692                 }
 693         }
 694
 695         if err != nil {
 696                 for k, v := range s.api.Logs {
 697                         c.Log(k)
 698                         c.Log(v.String())
 699                 }
 700         }
 701
 702         return s.api, s.runner, realTemp
 703 }
 704
 705 func (s *TestSuite) TestFullRunHello(c *C) {
 706         s.runner.enableMemoryLimit = true
 707         s.runner.networkMode = "default"
 708         s.fullRunHelper(c, `{
 709     "command": ["echo", "hello world"],
 710     "container_image": "`+arvadostest.DockerImage112PDH+`",
 711     "cwd": ".",
 712     "environment": {"foo":"bar","baz":"waz"},
 713     "mounts": {"/tmp": {"kind": "tmp"} },
 714     "output_path": "/tmp",
 715     "priority": 1,
 716     "runtime_constraints": {"vcpus":1,"ram":1000000},
 717     "state": "Locked",
 718     "output_storage_classes": ["default"]
 719 }`, nil, 0, func() {
 720                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 721                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 722                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 723                 c.Check(s.executor.created.VCPUs, Equals, 1)
 724                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 725                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 726                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 727                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 728         })
 729
 730         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 731         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 732         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 733         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 734         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 735 }
 736
 737 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 738         var ran bool
 739         s.fullRunHelper(c, `{
 740     "command": ["sleep", "3"],
 741     "container_image": "`+arvadostest.DockerImage112PDH+`",
 742     "cwd": ".",
 743     "environment": {},
 744     "mounts": {"/tmp": {"kind": "tmp"} },
 745     "output_path": "/tmp",
 746     "priority": 1,
 747     "runtime_constraints": {},
 748     "scheduling_parameters":{"max_run_time": 1},
 749     "state": "Running"
 750 }`, nil, 2, func() {
 751                 ran = true
 752         })
 753         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 754         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 755         c.Check(ran, Equals, false)
 756 }
 757
 758 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 759         s.fullRunHelper(c, `{
 760     "command": ["sleep", "3"],
 761     "container_image": "`+arvadostest.DockerImage112PDH+`",
 762     "cwd": ".",
 763     "environment": {},
 764     "mounts": {"/tmp": {"kind": "tmp"} },
 765     "output_path": "/tmp",
 766     "priority": 1,
 767     "runtime_constraints": {},
 768     "scheduling_parameters":{"max_run_time": 1},
 769     "state": "Locked"
 770 }`, nil, 0, func() {
 771                 time.Sleep(3 * time.Second)
 772         })
 773
 774         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 775         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 776 }
 777
 778 func (s *TestSuite) TestContainerWaitFails(c *C) {
 779         s.fullRunHelper(c, `{
 780     "command": ["sleep", "3"],
 781     "container_image": "`+arvadostest.DockerImage112PDH+`",
 782     "cwd": ".",
 783     "mounts": {"/tmp": {"kind": "tmp"} },
 784     "output_path": "/tmp",
 785     "priority": 1,
 786     "state": "Locked"
 787 }`, nil, 0, func() {
 788                 s.executor.waitErr = errors.New("Container is not running")
 789         })
 790
 791         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 792         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 793 }
 794
 795 func (s *TestSuite) TestCrunchstat(c *C) {
 796         s.fullRunHelper(c, `{
 797                 "command": ["sleep", "1"],
 798                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 799                 "cwd": ".",
 800                 "environment": {},
 801                 "mounts": {"/tmp": {"kind": "tmp"} },
 802                 "output_path": "/tmp",
 803                 "priority": 1,
 804                 "runtime_constraints": {},
 805                 "state": "Locked"
 806         }`, nil, 0, func() {
 807                 time.Sleep(time.Second)
 808         })
 809
 810         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 811         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 812
 813         // We didn't actually start a container, so crunchstat didn't
 814         // find accounting files and therefore didn't log any stats.
 815         // It should have logged a "can't find accounting files"
 816         // message after one poll interval, though, so we can confirm
 817         // it's alive:
 818         c.Assert(s.api.Logs["crunchstat"], NotNil)
 819         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 820
 821         // The "files never appeared" log assures us that we called
 822         // (*crunchstat.Reporter)Stop(), and that we set it up with
 823         // the correct container ID "abcde":
 824         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 825 }
 826
 827 func (s *TestSuite) TestNodeInfoLog(c *C) {
 828         os.Setenv("SLURMD_NODENAME", "compute2")
 829         s.fullRunHelper(c, `{
 830                 "command": ["sleep", "1"],
 831                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 832                 "cwd": ".",
 833                 "environment": {},
 834                 "mounts": {"/tmp": {"kind": "tmp"} },
 835                 "output_path": "/tmp",
 836                 "priority": 1,
 837                 "runtime_constraints": {},
 838                 "state": "Locked"
 839         }`, nil, 0,
 840                 func() {
 841                         time.Sleep(time.Second)
 842                 })
 843
 844         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 845         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 846
 847         c.Assert(s.api.Logs["node"], NotNil)
 848         json := s.api.Logs["node"].String()
 849         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 850         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 851         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 852
 853         c.Assert(s.api.Logs["node-info"], NotNil)
 854         json = s.api.Logs["node-info"].String()
 855         c.Check(json, Matches, `(?ms).*Host Information.*`)
 856         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 857         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 858         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 859         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 860 }
 861
 862 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 863         s.fullRunHelper(c, `{
 864                 "command": ["sleep", "1"],
 865                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 866                 "cwd": ".",
 867                 "environment": {},
 868                 "mounts": {"/tmp": {"kind": "tmp"} },
 869                 "output_path": "/tmp",
 870                 "priority": 1,
 871                 "runtime_constraints": {},
 872                 "state": "Locked"
 873         }`, nil, 0,
 874                 func() {
 875                 })
 876
 877         c.Assert(s.api.Logs["crunch-run"], NotNil)
 878         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 879         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 880 }
 881
 882 func (s *TestSuite) TestContainerRecordLog(c *C) {
 883         s.fullRunHelper(c, `{
 884                 "command": ["sleep", "1"],
 885                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 886                 "cwd": ".",
 887                 "environment": {},
 888                 "mounts": {"/tmp": {"kind": "tmp"} },
 889                 "output_path": "/tmp",
 890                 "priority": 1,
 891                 "runtime_constraints": {},
 892                 "state": "Locked"
 893         }`, nil, 0,
 894                 func() {
 895                         time.Sleep(time.Second)
 896                 })
 897
 898         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 899         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 900
 901         c.Assert(s.api.Logs["container"], NotNil)
 902         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 903 }
 904
 905 func (s *TestSuite) TestFullRunStderr(c *C) {
 906         s.fullRunHelper(c, `{
 907     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 908     "container_image": "`+arvadostest.DockerImage112PDH+`",
 909     "cwd": ".",
 910     "environment": {},
 911     "mounts": {"/tmp": {"kind": "tmp"} },
 912     "output_path": "/tmp",
 913     "priority": 1,
 914     "runtime_constraints": {},
 915     "state": "Locked"
 916 }`, nil, 1, func() {
 917                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 918                 fmt.Fprintln(s.executor.created.Stderr, "world")
 919         })
 920
 921         final := s.api.CalledWith("container.state", "Complete")
 922         c.Assert(final, NotNil)
 923         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 924         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 925
 926         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 927         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 928 }
 929
 930 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 931         s.fullRunHelper(c, `{
 932     "command": ["pwd"],
 933     "container_image": "`+arvadostest.DockerImage112PDH+`",
 934     "cwd": ".",
 935     "environment": {},
 936     "mounts": {"/tmp": {"kind": "tmp"} },
 937     "output_path": "/tmp",
 938     "priority": 1,
 939     "runtime_constraints": {},
 940     "state": "Locked"
 941 }`, nil, 0, func() {
 942                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 943         })
 944
 945         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 946         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 947         c.Log(s.api.Logs["stdout"])
 948         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 949 }
 950
 951 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 952         s.fullRunHelper(c, `{
 953     "command": ["pwd"],
 954     "container_image": "`+arvadostest.DockerImage112PDH+`",
 955     "cwd": "/bin",
 956     "environment": {},
 957     "mounts": {"/tmp": {"kind": "tmp"} },
 958     "output_path": "/tmp",
 959     "priority": 1,
 960     "runtime_constraints": {},
 961     "state": "Locked"
 962 }`, nil, 0, func() {
 963                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 964         })
 965
 966         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 967         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 968         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 969 }
 970
 971 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 972         s.fullRunHelper(c, `{
 973     "command": ["pwd"],
 974     "container_image": "`+arvadostest.DockerImage112PDH+`",
 975     "cwd": "/bin",
 976     "environment": {},
 977     "mounts": {"/tmp": {"kind": "tmp"} },
 978     "output_path": "/tmp",
 979     "priority": 1,
 980     "runtime_constraints": {},
 981     "state": "Locked",
 982     "output_storage_classes": ["foo", "bar"]
 983 }`, nil, 0, func() {
 984                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 985         })
 986
 987         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 988         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 989         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 990         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 991         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 992 }
 993
 994 func (s *TestSuite) TestStopOnSignal(c *C) {
 995         s.executor.runFunc = func() {
 996                 s.executor.created.Stdout.Write([]byte("foo\n"))
 997                 s.runner.SigChan <- syscall.SIGINT
 998         }
 999         s.testStopContainer(c)
1000 }
1001
1002 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1003         s.executor.runFunc = func() {
1004                 s.executor.created.Stdout.Write([]byte("foo\n"))
1005                 s.runner.ArvMountExit <- nil
1006                 close(s.runner.ArvMountExit)
1007         }
1008         s.runner.ArvMountExit = make(chan error)
1009         s.testStopContainer(c)
1010 }
1011
1012 func (s *TestSuite) testStopContainer(c *C) {
1013         record := `{
1014     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1015     "container_image": "` + arvadostest.DockerImage112PDH + `",
1016     "cwd": ".",
1017     "environment": {},
1018     "mounts": {"/tmp": {"kind": "tmp"} },
1019     "output_path": "/tmp",
1020     "priority": 1,
1021     "runtime_constraints": {},
1022     "state": "Locked"
1023 }`
1024
1025         err := json.Unmarshal([]byte(record), &s.api.Container)
1026         c.Assert(err, IsNil)
1027
1028         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1029         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1030                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1031         }
1032
1033         done := make(chan error)
1034         go func() {
1035                 done <- s.runner.Run()
1036         }()
1037         select {
1038         case <-time.After(20 * time.Second):
1039                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1040                 c.Fatal("timed out")
1041         case err = <-done:
1042                 c.Check(err, IsNil)
1043         }
1044         for k, v := range s.api.Logs {
1045                 c.Log(k)
1046                 c.Log(v.String(), "\n")
1047         }
1048
1049         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1050         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1051         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1052 }
1053
1054 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1055         s.fullRunHelper(c, `{
1056     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1057     "container_image": "`+arvadostest.DockerImage112PDH+`",
1058     "cwd": "/bin",
1059     "environment": {"FROBIZ": "bilbo"},
1060     "mounts": {"/tmp": {"kind": "tmp"} },
1061     "output_path": "/tmp",
1062     "priority": 1,
1063     "runtime_constraints": {},
1064     "state": "Locked"
1065 }`, nil, 0, func() {
1066                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1067         })
1068
1069         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1070         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1071         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1072 }
1073
1074 type ArvMountCmdLine struct {
1075         Cmd   []string
1076         token string
1077 }
1078
1079 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1080         am.Cmd = c
1081         am.token = token
1082         return nil, nil
1083 }
1084
1085 func stubCert(temp string) string {
1086         path := temp + "/ca-certificates.crt"
1087         crt, _ := os.Create(path)
1088         crt.Close()
1089         arvadosclient.CertFiles = []string{path}
1090         return path
1091 }
1092
1093 func (s *TestSuite) TestSetupMounts(c *C) {
1094         cr := s.runner
1095         am := &ArvMountCmdLine{}
1096         cr.RunArvMount = am.ArvMountTest
1097         cr.ContainerArvClient = &ArvTestClient{}
1098         cr.ContainerKeepClient = &KeepTestClient{}
1099         cr.Container.OutputStorageClasses = []string{"default"}
1100
1101         realTemp := c.MkDir()
1102         certTemp := c.MkDir()
1103         stubCertPath := stubCert(certTemp)
1104         cr.parentTemp = realTemp
1105
1106         i := 0
1107         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1108                 i++
1109                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1110                 err := os.Mkdir(d, os.ModePerm)
1111                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1112                         // Test case must have pre-populated the tempdir
1113                         err = nil
1114                 }
1115                 return d, err
1116         }
1117
1118         checkEmpty := func() {
1119                 // Should be deleted.
1120                 _, err := os.Stat(realTemp)
1121                 c.Assert(os.IsNotExist(err), Equals, true)
1122
1123                 // Now recreate it for the next test.
1124                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1125         }
1126
1127         {
1128                 i = 0
1129                 cr.ArvMountPoint = ""
1130                 cr.Container.Mounts = make(map[string]arvados.Mount)
1131                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1132                 cr.Container.OutputPath = "/tmp"
1133                 cr.statInterval = 5 * time.Second
1134                 bindmounts, err := cr.SetupMounts()
1135                 c.Check(err, IsNil)
1136                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1137                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1138                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1139                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1140                 os.RemoveAll(cr.ArvMountPoint)
1141                 cr.CleanupDirs()
1142                 checkEmpty()
1143         }
1144
1145         {
1146                 i = 0
1147                 cr.ArvMountPoint = ""
1148                 cr.Container.Mounts = make(map[string]arvados.Mount)
1149                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1150                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1151                 cr.Container.OutputPath = "/out"
1152                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1153
1154                 bindmounts, err := cr.SetupMounts()
1155                 c.Check(err, IsNil)
1156                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1157                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1158                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1159                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1160                 os.RemoveAll(cr.ArvMountPoint)
1161                 cr.CleanupDirs()
1162                 checkEmpty()
1163         }
1164
1165         {
1166                 i = 0
1167                 cr.ArvMountPoint = ""
1168                 cr.Container.Mounts = make(map[string]arvados.Mount)
1169                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1170                 cr.Container.OutputPath = "/tmp"
1171                 cr.Container.RuntimeConstraints.API = true
1172                 cr.Container.OutputStorageClasses = []string{"default"}
1173
1174                 bindmounts, err := cr.SetupMounts()
1175                 c.Check(err, IsNil)
1176                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1177                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1178                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1179                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1180                 os.RemoveAll(cr.ArvMountPoint)
1181                 cr.CleanupDirs()
1182                 checkEmpty()
1183
1184                 cr.Container.RuntimeConstraints.API = false
1185         }
1186
1187         {
1188                 i = 0
1189                 cr.ArvMountPoint = ""
1190                 cr.Container.Mounts = map[string]arvados.Mount{
1191                         "/keeptmp": {Kind: "collection", Writable: true},
1192                 }
1193                 cr.Container.OutputPath = "/keeptmp"
1194
1195                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1196
1197                 bindmounts, err := cr.SetupMounts()
1198                 c.Check(err, IsNil)
1199                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1200                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1201                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1202                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1203                 os.RemoveAll(cr.ArvMountPoint)
1204                 cr.CleanupDirs()
1205                 checkEmpty()
1206         }
1207
1208         {
1209                 i = 0
1210                 cr.ArvMountPoint = ""
1211                 cr.Container.Mounts = map[string]arvados.Mount{
1212                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1213                         "/keepout": {Kind: "collection", Writable: true},
1214                 }
1215                 cr.Container.OutputPath = "/keepout"
1216
1217                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1218                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1219
1220                 bindmounts, err := cr.SetupMounts()
1221                 c.Check(err, IsNil)
1222                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1223                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1224                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1225                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1226                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1227                         "/keepout": {realTemp + "/keep1/tmp0", false},
1228                 })
1229                 os.RemoveAll(cr.ArvMountPoint)
1230                 cr.CleanupDirs()
1231                 checkEmpty()
1232         }
1233
1234         {
1235                 i = 0
1236                 cr.ArvMountPoint = ""
1237                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1238                 cr.Container.Mounts = map[string]arvados.Mount{
1239                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1240                         "/keepout": {Kind: "collection", Writable: true},
1241                 }
1242                 cr.Container.OutputPath = "/keepout"
1243
1244                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1245                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1246
1247                 bindmounts, err := cr.SetupMounts()
1248                 c.Check(err, IsNil)
1249                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1250                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1251                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1252                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1253                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1254                         "/keepout": {realTemp + "/keep1/tmp0", false},
1255                 })
1256                 os.RemoveAll(cr.ArvMountPoint)
1257                 cr.CleanupDirs()
1258                 checkEmpty()
1259         }
1260
1261         for _, test := range []struct {
1262                 in  interface{}
1263                 out string
1264         }{
1265                 {in: "foo", out: `"foo"`},
1266                 {in: nil, out: `null`},
1267                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1268         } {
1269                 i = 0
1270                 cr.ArvMountPoint = ""
1271                 cr.Container.Mounts = map[string]arvados.Mount{
1272                         "/mnt/test.json": {Kind: "json", Content: test.in},
1273                 }
1274                 bindmounts, err := cr.SetupMounts()
1275                 c.Check(err, IsNil)
1276                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1277                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1278                 })
1279                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1280                 c.Check(err, IsNil)
1281                 c.Check(content, DeepEquals, []byte(test.out))
1282                 os.RemoveAll(cr.ArvMountPoint)
1283                 cr.CleanupDirs()
1284                 checkEmpty()
1285         }
1286
1287         for _, test := range []struct {
1288                 in  interface{}
1289                 out string
1290         }{
1291                 {in: "foo", out: `foo`},
1292                 {in: nil, out: "error"},
1293                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1294         } {
1295                 i = 0
1296                 cr.ArvMountPoint = ""
1297                 cr.Container.Mounts = map[string]arvados.Mount{
1298                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1299                 }
1300                 bindmounts, err := cr.SetupMounts()
1301                 if test.out == "error" {
1302                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1303                 } else {
1304                         c.Check(err, IsNil)
1305                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1306                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1307                         })
1308                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1309                         c.Check(err, IsNil)
1310                         c.Check(content, DeepEquals, []byte(test.out))
1311                 }
1312                 os.RemoveAll(cr.ArvMountPoint)
1313                 cr.CleanupDirs()
1314                 checkEmpty()
1315         }
1316
1317         // Read-only mount points are allowed underneath output_dir mount point
1318         {
1319                 i = 0
1320                 cr.ArvMountPoint = ""
1321                 cr.Container.Mounts = make(map[string]arvados.Mount)
1322                 cr.Container.Mounts = map[string]arvados.Mount{
1323                         "/tmp":     {Kind: "tmp"},
1324                         "/tmp/foo": {Kind: "collection"},
1325                 }
1326                 cr.Container.OutputPath = "/tmp"
1327
1328                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1329
1330                 bindmounts, err := cr.SetupMounts()
1331                 c.Check(err, IsNil)
1332                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1333                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1334                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1335                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1336                         "/tmp":     {realTemp + "/tmp2", false},
1337                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1338                 })
1339                 os.RemoveAll(cr.ArvMountPoint)
1340                 cr.CleanupDirs()
1341                 checkEmpty()
1342         }
1343
1344         // Writable mount points copied to output_dir mount point
1345         {
1346                 i = 0
1347                 cr.ArvMountPoint = ""
1348                 cr.Container.Mounts = make(map[string]arvados.Mount)
1349                 cr.Container.Mounts = map[string]arvados.Mount{
1350                         "/tmp": {Kind: "tmp"},
1351                         "/tmp/foo": {Kind: "collection",
1352                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1353                                 Writable:         true},
1354                         "/tmp/bar": {Kind: "collection",
1355                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1356                                 Path:             "baz",
1357                                 Writable:         true},
1358                 }
1359                 cr.Container.OutputPath = "/tmp"
1360
1361                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1362                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1363
1364                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1365                 rf.Write([]byte("bar"))
1366                 rf.Close()
1367
1368                 _, err := cr.SetupMounts()
1369                 c.Check(err, IsNil)
1370                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1371                 c.Check(err, IsNil)
1372                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1373                 c.Check(err, IsNil)
1374                 os.RemoveAll(cr.ArvMountPoint)
1375                 cr.CleanupDirs()
1376                 checkEmpty()
1377         }
1378
1379         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1380         {
1381                 i = 0
1382                 cr.ArvMountPoint = ""
1383                 cr.Container.Mounts = make(map[string]arvados.Mount)
1384                 cr.Container.Mounts = map[string]arvados.Mount{
1385                         "/tmp":     {Kind: "tmp"},
1386                         "/tmp/foo": {Kind: "tmp"},
1387                 }
1388                 cr.Container.OutputPath = "/tmp"
1389
1390                 _, err := cr.SetupMounts()
1391                 c.Check(err, NotNil)
1392                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1393                 os.RemoveAll(cr.ArvMountPoint)
1394                 cr.CleanupDirs()
1395                 checkEmpty()
1396         }
1397
1398         // Only mount point of kind 'collection' is allowed for stdin
1399         {
1400                 i = 0
1401                 cr.ArvMountPoint = ""
1402                 cr.Container.Mounts = make(map[string]arvados.Mount)
1403                 cr.Container.Mounts = map[string]arvados.Mount{
1404                         "stdin": {Kind: "tmp"},
1405                 }
1406
1407                 _, err := cr.SetupMounts()
1408                 c.Check(err, NotNil)
1409                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1410                 os.RemoveAll(cr.ArvMountPoint)
1411                 cr.CleanupDirs()
1412                 checkEmpty()
1413         }
1414
1415         // git_tree mounts
1416         {
1417                 i = 0
1418                 cr.ArvMountPoint = ""
1419                 (*GitMountSuite)(nil).useTestGitServer(c)
1420                 cr.token = arvadostest.ActiveToken
1421                 cr.Container.Mounts = make(map[string]arvados.Mount)
1422                 cr.Container.Mounts = map[string]arvados.Mount{
1423                         "/tip": {
1424                                 Kind:   "git_tree",
1425                                 UUID:   arvadostest.Repository2UUID,
1426                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1427                                 Path:   "/",
1428                         },
1429                         "/non-tip": {
1430                                 Kind:   "git_tree",
1431                                 UUID:   arvadostest.Repository2UUID,
1432                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1433                                 Path:   "/",
1434                         },
1435                 }
1436                 cr.Container.OutputPath = "/tmp"
1437
1438                 bindmounts, err := cr.SetupMounts()
1439                 c.Check(err, IsNil)
1440
1441                 for path, mount := range bindmounts {
1442                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1443                 }
1444
1445                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1446                 c.Check(err, IsNil)
1447                 c.Check(string(data), Equals, "\000\001\002\003")
1448                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1449                 c.Check(err, FitsTypeOf, &os.PathError{})
1450                 c.Check(os.IsNotExist(err), Equals, true)
1451
1452                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1453                 c.Check(err, IsNil)
1454                 c.Check(string(data), Equals, "\000\001\002\003")
1455                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1456                 c.Check(err, IsNil)
1457                 c.Check(string(data), Equals, "testfile\n")
1458
1459                 cr.CleanupDirs()
1460                 checkEmpty()
1461         }
1462 }
1463
1464 func (s *TestSuite) TestStdout(c *C) {
1465         helperRecord := `{
1466                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1467                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1468                 "cwd": "/bin",
1469                 "environment": {"FROBIZ": "bilbo"},
1470                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1471                 "output_path": "/tmp",
1472                 "priority": 1,
1473                 "runtime_constraints": {},
1474                 "state": "Locked"
1475         }`
1476
1477         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1478                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1479         })
1480
1481         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1482         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1483         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1484 }
1485
1486 // Used by the TestStdoutWithWrongPath*()
1487 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1488         err := json.Unmarshal([]byte(record), &s.api.Container)
1489         c.Assert(err, IsNil)
1490         s.executor.runFunc = fn
1491         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1492         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1493                 return s.api, &KeepTestClient{}, nil, nil
1494         }
1495         return s.api, s.runner, s.runner.Run()
1496 }
1497
1498 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1499         _, _, err := s.stdoutErrorRunHelper(c, `{
1500     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1501     "output_path": "/tmp",
1502     "state": "Locked"
1503 }`, func() {})
1504         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1505 }
1506
1507 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1508         _, _, err := s.stdoutErrorRunHelper(c, `{
1509     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1510     "output_path": "/tmp",
1511     "state": "Locked"
1512 }`, func() {})
1513         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1514 }
1515
1516 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1517         _, _, err := s.stdoutErrorRunHelper(c, `{
1518     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1519     "output_path": "/tmp",
1520     "state": "Locked"
1521 }`, func() {})
1522         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1523 }
1524
1525 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1526         s.fullRunHelper(c, `{
1527     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1528     "container_image": "`+arvadostest.DockerImage112PDH+`",
1529     "cwd": "/bin",
1530     "environment": {},
1531     "mounts": {"/tmp": {"kind": "tmp"} },
1532     "output_path": "/tmp",
1533     "priority": 1,
1534     "runtime_constraints": {"API": true},
1535     "state": "Locked"
1536 }`, nil, 0, func() {
1537                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1538                 s.executor.exit <- 3
1539         })
1540         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1541         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1542 }
1543
1544 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1545         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1546         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1547         s.fullRunHelper(c, `{
1548     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1549     "container_image": "`+arvadostest.DockerImage112PDH+`",
1550     "cwd": "/bin",
1551     "environment": {},
1552     "mounts": {"/tmp": {"kind": "tmp"} },
1553     "output_path": "/tmp",
1554     "priority": 1,
1555     "runtime_constraints": {"API": true},
1556     "state": "Locked"
1557 }`, nil, 0, func() {
1558                 s.api.Container.Output = arvadostest.DockerImage112PDH
1559         })
1560
1561         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1562         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1563         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1564 }
1565
1566 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1567         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1568                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1569                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1570                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1571         }
1572         s.executor.runFunc = func() {
1573                 time.Sleep(time.Second)
1574                 s.executor.exit <- 0
1575         }
1576         record := `{
1577     "command": ["sleep", "1"],
1578     "container_image": "` + arvadostest.DockerImage112PDH + `",
1579     "cwd": "/bin",
1580     "environment": {},
1581     "mounts": {"/tmp": {"kind": "tmp"} },
1582     "output_path": "/tmp",
1583     "priority": 1,
1584     "runtime_constraints": {"API": true},
1585     "state": "Locked"
1586 }`
1587         err := json.Unmarshal([]byte(record), &s.api.Container)
1588         c.Assert(err, IsNil)
1589         err = s.runner.Run()
1590         c.Assert(err, IsNil)
1591         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1592         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1593         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1594         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1595 }
1596
1597 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1598         helperRecord := `{
1599                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1600                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1601                 "cwd": "/bin",
1602                 "environment": {"FROBIZ": "bilbo"},
1603                 "mounts": {
1604         "/tmp": {"kind": "tmp"},
1605         "/tmp/foo": {"kind": "collection",
1606                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1607                      "exclude_from_output": true
1608         },
1609         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1610     },
1611                 "output_path": "/tmp",
1612                 "priority": 1,
1613                 "runtime_constraints": {},
1614                 "state": "Locked"
1615         }`
1616
1617         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1618
1619         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1620                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1621         })
1622
1623         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1624         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1625         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1626 }
1627
1628 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1629         helperRecord := `{
1630                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1631                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1632                 "cwd": "/bin",
1633                 "environment": {"FROBIZ": "bilbo"},
1634                 "mounts": {
1635         "/tmp": {"kind": "tmp"},
1636         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1637         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1638         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1639         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1640         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1641     },
1642                 "output_path": "/tmp",
1643                 "priority": 1,
1644                 "runtime_constraints": {},
1645                 "state": "Locked"
1646         }`
1647
1648         extraMounts := []string{
1649                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1650                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1651                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1652         }
1653
1654         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1655                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1656         })
1657
1658         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1659                 "/tmp":                   {realtemp + "/tmp1", false},
1660                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1661                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1662                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1663                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1664         })
1665
1666         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1667         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1668         for _, v := range api.Content {
1669                 if v["collection"] != nil {
1670                         c.Check(v["ensure_unique_name"], Equals, true)
1671                         collection := v["collection"].(arvadosclient.Dict)
1672                         if strings.Index(collection["name"].(string), "output") == 0 {
1673                                 manifest := collection["manifest_text"].(string)
1674
1675                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1676 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1677 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1678 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1679 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1680 `)
1681                         }
1682                 }
1683         }
1684 }
1685
1686 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1687         helperRecord := `{
1688                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1689                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1690                 "cwd": "/bin",
1691                 "environment": {"FROBIZ": "bilbo"},
1692                 "mounts": {
1693         "/tmp": {"kind": "tmp"},
1694         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1695         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1696     },
1697                 "output_path": "/tmp",
1698                 "priority": 1,
1699                 "runtime_constraints": {},
1700                 "state": "Locked"
1701         }`
1702
1703         extraMounts := []string{
1704                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1705         }
1706
1707         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1708                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1709         })
1710
1711         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1712         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1713         for _, v := range s.api.Content {
1714                 if v["collection"] != nil {
1715                         collection := v["collection"].(arvadosclient.Dict)
1716                         if strings.Index(collection["name"].(string), "output") == 0 {
1717                                 manifest := collection["manifest_text"].(string)
1718
1719                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1720 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1721 `)
1722                         }
1723                 }
1724         }
1725 }
1726
1727 func (s *TestSuite) TestOutputError(c *C) {
1728         helperRecord := `{
1729                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1730                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1731                 "cwd": "/bin",
1732                 "environment": {"FROBIZ": "bilbo"},
1733                 "mounts": {
1734                         "/tmp": {"kind": "tmp"}
1735                 },
1736                 "output_path": "/tmp",
1737                 "priority": 1,
1738                 "runtime_constraints": {},
1739                 "state": "Locked"
1740         }`
1741         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1742                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1743         })
1744
1745         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1746 }
1747
1748 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1749         helperRecord := `{
1750                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1751                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1752                 "cwd": "/bin",
1753                 "environment": {"FROBIZ": "bilbo"},
1754                 "mounts": {
1755         "/tmp": {"kind": "tmp"},
1756         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1757         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1758     },
1759                 "output_path": "/tmp",
1760                 "priority": 1,
1761                 "runtime_constraints": {},
1762                 "state": "Locked"
1763         }`
1764
1765         extraMounts := []string{
1766                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1767         }
1768
1769         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1770                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1771         })
1772
1773         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1774         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1775         for _, v := range api.Content {
1776                 if v["collection"] != nil {
1777                         collection := v["collection"].(arvadosclient.Dict)
1778                         if strings.Index(collection["name"].(string), "output") == 0 {
1779                                 manifest := collection["manifest_text"].(string)
1780                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1781 `)
1782                         }
1783                 }
1784         }
1785 }
1786
1787 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1788         helperRecord := `{
1789                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1790                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1791                 "cwd": "/bin",
1792                 "environment": {"FROBIZ": "bilbo"},
1793                 "mounts": {
1794         "/tmp": {"kind": "tmp"},
1795         "stdin": {"kind": "json", "content": "foo"},
1796         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1797     },
1798                 "output_path": "/tmp",
1799                 "priority": 1,
1800                 "runtime_constraints": {},
1801                 "state": "Locked"
1802         }`
1803
1804         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1805                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1806         })
1807
1808         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1809         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1810         for _, v := range api.Content {
1811                 if v["collection"] != nil {
1812                         collection := v["collection"].(arvadosclient.Dict)
1813                         if strings.Index(collection["name"].(string), "output") == 0 {
1814                                 manifest := collection["manifest_text"].(string)
1815                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1816 `)
1817                         }
1818                 }
1819         }
1820 }
1821
1822 func (s *TestSuite) TestStderrMount(c *C) {
1823         api, cr, _ := s.fullRunHelper(c, `{
1824     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1825     "container_image": "`+arvadostest.DockerImage112PDH+`",
1826     "cwd": ".",
1827     "environment": {},
1828     "mounts": {"/tmp": {"kind": "tmp"},
1829                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1830                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1831     "output_path": "/tmp",
1832     "priority": 1,
1833     "runtime_constraints": {},
1834     "state": "Locked"
1835 }`, nil, 1, func() {
1836                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1837                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1838         })
1839
1840         final := api.CalledWith("container.state", "Complete")
1841         c.Assert(final, NotNil)
1842         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1843         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1844
1845         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1846 }
1847
1848 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1849         s.api.callraw = true
1850         err := s.runner.fetchContainerRecord()
1851         c.Assert(err, IsNil)
1852         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1853         c.Logf("%#v", s.runner.Container)
1854         c.Check(err, IsNil)
1855         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1856 }
1857
1858 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1859         nextState := ""
1860         for _, setup := range []func(){
1861                 func() {
1862                         c.Log("// waitErr = ocl runtime error")
1863                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1864                         nextState = "Cancelled"
1865                 },
1866                 func() {
1867                         c.Log("// loadErr = cannot connect")
1868                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1869                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1870                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1871                         c.Assert(err, IsNil)
1872                         nextState = "Queued"
1873                 },
1874         } {
1875                 s.SetUpTest(c)
1876                 setup()
1877                 s.fullRunHelper(c, `{
1878     "command": ["echo", "hello world"],
1879     "container_image": "`+arvadostest.DockerImage112PDH+`",
1880     "cwd": ".",
1881     "environment": {},
1882     "mounts": {"/tmp": {"kind": "tmp"} },
1883     "output_path": "/tmp",
1884     "priority": 1,
1885     "runtime_constraints": {},
1886     "state": "Locked"
1887 }`, nil, 0, func() {})
1888                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1889                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1890                 if *brokenNodeHook != "" {
1891                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1892                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1893                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1894                 } else {
1895                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1896                 }
1897         }
1898 }
1899
1900 func (s *TestSuite) TestBadCommand(c *C) {
1901         for _, startError := range []string{
1902                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1903                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1904                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1905         } {
1906                 s.SetUpTest(c)
1907                 s.executor.startErr = errors.New(startError)
1908                 s.fullRunHelper(c, `{
1909     "command": ["echo", "hello world"],
1910     "container_image": "`+arvadostest.DockerImage112PDH+`",
1911     "cwd": ".",
1912     "environment": {},
1913     "mounts": {"/tmp": {"kind": "tmp"} },
1914     "output_path": "/tmp",
1915     "priority": 1,
1916     "runtime_constraints": {},
1917     "state": "Locked"
1918 }`, nil, 0, func() {})
1919                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1920                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1921         }
1922 }
1923
1924 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1925         helperRecord := `{
1926                 "command": ["true"],
1927                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1928                 "cwd": "/bin",
1929                 "mounts": {
1930                     "/tmp": {"kind": "tmp"},
1931                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1932                 },
1933                 "secret_mounts": {
1934                 },
1935                 "output_path": "/tmp",
1936                 "priority": 1,
1937                 "runtime_constraints": {},
1938                 "state": "Locked"
1939         }`
1940
1941         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1942                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1943                 c.Check(err, IsNil)
1944                 c.Check(string(content), Equals, "mypassword")
1945         })
1946
1947         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1948         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1949         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1950         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1951
1952         // under secret mounts, not captured in output
1953         helperRecord = `{
1954                 "command": ["true"],
1955                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1956                 "cwd": "/bin",
1957                 "mounts": {
1958                     "/tmp": {"kind": "tmp"}
1959                 },
1960                 "secret_mounts": {
1961                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1962                 },
1963                 "output_path": "/tmp",
1964                 "priority": 1,
1965                 "runtime_constraints": {},
1966                 "state": "Locked"
1967         }`
1968
1969         s.SetUpTest(c)
1970         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1971                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1972                 c.Check(err, IsNil)
1973                 c.Check(string(content), Equals, "mypassword")
1974         })
1975
1976         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1977         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1978         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1979         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1980
1981         // under secret mounts, output dir is a collection, not captured in output
1982         helperRecord = `{
1983                 "command": ["true"],
1984                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1985                 "cwd": "/bin",
1986                 "mounts": {
1987                     "/tmp": {"kind": "collection", "writable": true}
1988                 },
1989                 "secret_mounts": {
1990                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1991                 },
1992                 "output_path": "/tmp",
1993                 "priority": 1,
1994                 "runtime_constraints": {},
1995                 "state": "Locked"
1996         }`
1997
1998         s.SetUpTest(c)
1999         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2000                 // secret.conf should be provisioned as a separate
2001                 // bind mount, i.e., it should not appear in the
2002                 // (fake) fuse filesystem as viewed from the host.
2003                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2004                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2005                         c.Logf("secret.conf: content %q, err %#v", content, err)
2006                 }
2007                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2008                 c.Check(err, IsNil)
2009         })
2010
2011         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2012         c.Check(err, IsNil)
2013         c.Check(string(content), Equals, "mypassword")
2014         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2015         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2016         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2017         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2018 }
2019
2020 type FakeProcess struct {
2021         cmdLine []string
2022 }
2023
2024 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2025         return fp.cmdLine, nil
2026 }