lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 116         containerClient *arvados.Client) error {
 117         e.loaded = tarball
 118         return e.loadErr
 119 }
 120 func (e *stubExecutor) Runtime() string                 { return "stub" }
 121 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 122 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 123 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 124 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 125 func (e *stubExecutor) Close()                          { e.closed = true }
 126 func (e *stubExecutor) Wait(context.Context) (int, error) {
 127         return <-e.exit, e.waitErr
 128 }
 129
 130 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 131
 132 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 133 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 134 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 135
 136 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 137 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 138
 139 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 140 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 141 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 142 `
 143
 144 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 145
 146 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 147 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 148
 149 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 150 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 151
 152 func (client *ArvTestClient) Create(resourceType string,
 153         parameters arvadosclient.Dict,
 154         output interface{}) error {
 155
 156         client.Mutex.Lock()
 157         defer client.Mutex.Unlock()
 158
 159         client.Calls++
 160         client.Content = append(client.Content, parameters)
 161
 162         if resourceType == "logs" {
 163                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 164                 if client.Logs == nil {
 165                         client.Logs = make(map[string]*bytes.Buffer)
 166                 }
 167                 if client.Logs[et] == nil {
 168                         client.Logs[et] = &bytes.Buffer{}
 169                 }
 170                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 171         }
 172
 173         if resourceType == "collections" && output != nil {
 174                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 175                 outmap := output.(*arvados.Collection)
 176                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 177                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 178         }
 179
 180         return nil
 181 }
 182
 183 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 184         switch {
 185         case method == "GET" && resourceType == "containers" && action == "auth":
 186                 return json.Unmarshal([]byte(`{
 187                         "kind": "arvados#api_client_authorization",
 188                         "uuid": "`+fakeAuthUUID+`",
 189                         "api_token": "`+fakeAuthToken+`"
 190                         }`), output)
 191         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 192                 if client.secretMounts != nil {
 193                         return json.Unmarshal(client.secretMounts, output)
 194                 }
 195                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 196         default:
 197                 return fmt.Errorf("Not found")
 198         }
 199 }
 200
 201 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 202         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 203         var j []byte
 204         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 205                 j = []byte(`{
 206                         "kind": "arvados#nodeList",
 207                         "items": [{
 208                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 209                                 "hostname": "compute2",
 210                                 "properties": {"total_cpu_cores": 16}
 211                         }]}`)
 212         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 213                 if uuid == "" {
 214                         j, err = json.Marshal(map[string]interface{}{
 215                                 "items": []interface{}{client.Container},
 216                                 "kind":  "arvados#nodeList",
 217                         })
 218                 } else {
 219                         j, err = json.Marshal(client.Container)
 220                 }
 221         } else {
 222                 j = []byte(`{
 223                         "command": ["sleep", "1"],
 224                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 225                         "cwd": ".",
 226                         "environment": {},
 227                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 228                         "output_path": "/tmp",
 229                         "priority": 1,
 230                         "runtime_constraints": {}
 231                 }`)
 232         }
 233         return ioutil.NopCloser(bytes.NewReader(j)), err
 234 }
 235
 236 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 237         if resourceType == "collections" {
 238                 if uuid == hwPDH {
 239                         output.(*arvados.Collection).ManifestText = hwManifest
 240                 } else if uuid == otherPDH {
 241                         output.(*arvados.Collection).ManifestText = otherManifest
 242                 } else if uuid == normalizedWithSubdirsPDH {
 243                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 244                 } else if uuid == denormalizedWithSubdirsPDH {
 245                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 246                 }
 247         }
 248         if resourceType == "containers" {
 249                 (*output.(*arvados.Container)) = client.Container
 250         }
 251         return nil
 252 }
 253
 254 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 255         client.Mutex.Lock()
 256         defer client.Mutex.Unlock()
 257         client.Calls++
 258         client.Content = append(client.Content, parameters)
 259         if resourceType == "containers" {
 260                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 261                         client.WasSetRunning = true
 262                 }
 263         } else if resourceType == "collections" {
 264                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 265                 output.(*arvados.Collection).UUID = uuid
 266                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 267         }
 268         return nil
 269 }
 270
 271 var discoveryMap = map[string]interface{}{
 272         "defaultTrashLifetime":               float64(1209600),
 273         "crunchLimitLogBytesPerJob":          float64(67108864),
 274         "crunchLogThrottleBytes":             float64(65536),
 275         "crunchLogThrottlePeriod":            float64(60),
 276         "crunchLogThrottleLines":             float64(1024),
 277         "crunchLogPartialLineThrottlePeriod": float64(5),
 278         "crunchLogBytesPerEvent":             float64(4096),
 279         "crunchLogSecondsBetweenEvents":      float64(1),
 280 }
 281
 282 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 283         return discoveryMap[key], nil
 284 }
 285
 286 // CalledWith returns the parameters from the first API call whose
 287 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 288 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 289 // no call matches, it returns nil.
 290 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 291 call:
 292         for _, content := range client.Content {
 293                 var v interface{} = content
 294                 for _, k := range strings.Split(jpath, ".") {
 295                         if dict, ok := v.(arvadosclient.Dict); !ok {
 296                                 continue call
 297                         } else {
 298                                 v = dict[k]
 299                         }
 300                 }
 301                 if v == expect {
 302                         return content
 303                 }
 304         }
 305         return nil
 306 }
 307
 308 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 309         return locator, nil
 310 }
 311
 312 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 313         client.Content = opts.Data
 314         return arvados.BlockWriteResponse{
 315                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 316         }, nil
 317 }
 318
 319 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 320         return 0, errors.New("not implemented")
 321 }
 322
 323 func (client *KeepTestClient) ClearBlockCache() {
 324 }
 325
 326 func (client *KeepTestClient) Close() {
 327         client.Content = nil
 328 }
 329
 330 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 331         client.StorageClasses = sc
 332 }
 333
 334 type FileWrapper struct {
 335         io.ReadCloser
 336         len int64
 337 }
 338
 339 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 340         return nil, errors.New("not implemented")
 341 }
 342
 343 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 344         return 0, errors.New("not implemented")
 345 }
 346
 347 func (fw FileWrapper) Size() int64 {
 348         return fw.len
 349 }
 350
 351 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 352         return nil, errors.New("not implemented")
 353 }
 354
 355 func (fw FileWrapper) Truncate(int64) error {
 356         return errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Write([]byte) (int, error) {
 360         return 0, errors.New("not implemented")
 361 }
 362
 363 func (fw FileWrapper) Sync() error {
 364         return errors.New("not implemented")
 365 }
 366
 367 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 368         return nil, errors.New("not implemented")
 369 }
 370
 371 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 372         return errors.New("not implemented")
 373 }
 374
 375 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 376         if filename == hwImageID+".tar" {
 377                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 378                 client.Called = true
 379                 return FileWrapper{rdr, 1321984}, nil
 380         } else if filename == "/file1_in_main.txt" {
 381                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 382                 client.Called = true
 383                 return FileWrapper{rdr, 3}, nil
 384         }
 385         return nil, nil
 386 }
 387
 388 func (s *TestSuite) TestLoadImage(c *C) {
 389         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 390         s.runner.Container.Mounts = map[string]arvados.Mount{
 391                 "/out": {Kind: "tmp", Writable: true},
 392         }
 393         s.runner.Container.OutputPath = "/out"
 394
 395         _, err := s.runner.SetupMounts()
 396         c.Assert(err, IsNil)
 397
 398         imageID, err := s.runner.LoadImage()
 399         c.Check(err, IsNil)
 400         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 401         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 402
 403         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 404         s.executor.imageLoaded = false
 405         s.executor.loaded = ""
 406         s.executor.loadErr = errors.New("bork")
 407         imageID, err = s.runner.LoadImage()
 408         c.Check(err, ErrorMatches, ".*bork")
 409         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 410
 411         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 412         s.executor.imageLoaded = false
 413         s.executor.loaded = ""
 414         s.executor.loadErr = nil
 415         imageID, err = s.runner.LoadImage()
 416         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 417         c.Check(s.executor.loaded, Equals, "")
 418 }
 419
 420 type ArvErrorTestClient struct{}
 421
 422 func (ArvErrorTestClient) Create(resourceType string,
 423         parameters arvadosclient.Dict,
 424         output interface{}) error {
 425         return nil
 426 }
 427
 428 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 429         if method == "GET" && resourceType == "containers" && action == "auth" {
 430                 return nil
 431         }
 432         return errors.New("ArvError")
 433 }
 434
 435 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 436         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 437         return nil, errors.New("ArvError")
 438 }
 439
 440 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 441         return errors.New("ArvError")
 442 }
 443
 444 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 445         return nil
 446 }
 447
 448 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 449         return discoveryMap[key], nil
 450 }
 451
 452 type KeepErrorTestClient struct {
 453         KeepTestClient
 454 }
 455
 456 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 457         return nil, errors.New("KeepError")
 458 }
 459
 460 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 461         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 462 }
 463
 464 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 465         return "", errors.New("KeepError")
 466 }
 467
 468 type KeepReadErrorTestClient struct {
 469         KeepTestClient
 470 }
 471
 472 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 473         return 0, errors.New("KeepError")
 474 }
 475
 476 type ErrorReader struct {
 477         FileWrapper
 478 }
 479
 480 func (ErrorReader) Read(p []byte) (n int, err error) {
 481         return 0, errors.New("ErrorReader")
 482 }
 483
 484 func (ErrorReader) Seek(int64, int) (int64, error) {
 485         return 0, errors.New("ErrorReader")
 486 }
 487
 488 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 489         return ErrorReader{}, nil
 490 }
 491
 492 type ClosableBuffer struct {
 493         bytes.Buffer
 494 }
 495
 496 func (*ClosableBuffer) Close() error {
 497         return nil
 498 }
 499
 500 type TestLogs struct {
 501         Stdout ClosableBuffer
 502         Stderr ClosableBuffer
 503 }
 504
 505 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 506         if logstr == "stdout" {
 507                 return &tl.Stdout, nil
 508         }
 509         if logstr == "stderr" {
 510                 return &tl.Stderr, nil
 511         }
 512         return nil, errors.New("???")
 513 }
 514
 515 func dockerLog(fd byte, msg string) []byte {
 516         by := []byte(msg)
 517         header := make([]byte, 8+len(by))
 518         header[0] = fd
 519         header[7] = byte(len(by))
 520         copy(header[8:], by)
 521         return header
 522 }
 523
 524 func (s *TestSuite) TestRunContainer(c *C) {
 525         s.executor.runFunc = func() {
 526                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 527                 s.executor.exit <- 0
 528         }
 529
 530         var logs TestLogs
 531         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 532         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 533         s.runner.Container.Command = []string{"./hw"}
 534         s.runner.Container.OutputStorageClasses = []string{"default"}
 535
 536         imageID, err := s.runner.LoadImage()
 537         c.Assert(err, IsNil)
 538
 539         err = s.runner.CreateContainer(imageID, nil)
 540         c.Assert(err, IsNil)
 541
 542         err = s.runner.StartContainer()
 543         c.Assert(err, IsNil)
 544
 545         err = s.runner.WaitFinish()
 546         c.Assert(err, IsNil)
 547
 548         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 549         c.Check(logs.Stderr.String(), Equals, "")
 550 }
 551
 552 func (s *TestSuite) TestCommitLogs(c *C) {
 553         api := &ArvTestClient{}
 554         kc := &KeepTestClient{}
 555         defer kc.Close()
 556         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 557         c.Assert(err, IsNil)
 558         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 559
 560         cr.CrunchLog.Print("Hello world!")
 561         cr.CrunchLog.Print("Goodbye")
 562         cr.finalState = "Complete"
 563
 564         err = cr.CommitLogs()
 565         c.Check(err, IsNil)
 566
 567         c.Check(api.Calls, Equals, 2)
 568         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 569         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 570         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 571         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 572 }
 573
 574 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 575         api := &ArvTestClient{}
 576         kc := &KeepTestClient{}
 577         defer kc.Close()
 578         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 579         c.Assert(err, IsNil)
 580
 581         err = cr.UpdateContainerRunning()
 582         c.Check(err, IsNil)
 583
 584         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 585 }
 586
 587 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 588         api := &ArvTestClient{}
 589         kc := &KeepTestClient{}
 590         defer kc.Close()
 591         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 592         c.Assert(err, IsNil)
 593
 594         cr.LogsPDH = new(string)
 595         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 596
 597         cr.ExitCode = new(int)
 598         *cr.ExitCode = 42
 599         cr.finalState = "Complete"
 600
 601         err = cr.UpdateContainerFinal()
 602         c.Check(err, IsNil)
 603
 604         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 605         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 606         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 607 }
 608
 609 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 610         api := &ArvTestClient{}
 611         kc := &KeepTestClient{}
 612         defer kc.Close()
 613         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 614         c.Assert(err, IsNil)
 615         cr.cCancelled = true
 616         cr.finalState = "Cancelled"
 617
 618         err = cr.UpdateContainerFinal()
 619         c.Check(err, IsNil)
 620
 621         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 622         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 624 }
 625
 626 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 627 // dress rehearsal of the Run() function, starting from a JSON container record.
 628 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 629         err := json.Unmarshal([]byte(record), &s.api.Container)
 630         c.Assert(err, IsNil)
 631         initialState := s.api.Container.State
 632
 633         var sm struct {
 634                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 635         }
 636         err = json.Unmarshal([]byte(record), &sm)
 637         c.Check(err, IsNil)
 638         secretMounts, err := json.Marshal(sm)
 639         c.Assert(err, IsNil)
 640         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 641
 642         s.executor.runFunc = func() {
 643                 fn()
 644                 s.executor.exit <- exitCode
 645         }
 646
 647         s.runner.statInterval = 100 * time.Millisecond
 648         s.runner.containerWatchdogInterval = time.Second
 649         am := &ArvMountCmdLine{}
 650         s.runner.RunArvMount = am.ArvMountTest
 651
 652         realTemp := c.MkDir()
 653         tempcount := 0
 654         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 655                 tempcount++
 656                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 657                 err := os.Mkdir(d, os.ModePerm)
 658                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 659                         // Test case must have pre-populated the tempdir
 660                         err = nil
 661                 }
 662                 return d, err
 663         }
 664         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 665                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 666         }
 667
 668         if extraMounts != nil && len(extraMounts) > 0 {
 669                 err := s.runner.SetupArvMountPoint("keep")
 670                 c.Check(err, IsNil)
 671
 672                 for _, m := range extraMounts {
 673                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 674                 }
 675         }
 676
 677         err = s.runner.Run()
 678         if s.api.CalledWith("container.state", "Complete") != nil {
 679                 c.Check(err, IsNil)
 680         }
 681         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 682                 c.Check(s.api.WasSetRunning, Equals, true)
 683                 var lastupdate arvadosclient.Dict
 684                 for _, content := range s.api.Content {
 685                         if content["container"] != nil {
 686                                 lastupdate = content["container"].(arvadosclient.Dict)
 687                         }
 688                 }
 689                 if lastupdate["log"] == nil {
 690                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 691                 }
 692         }
 693
 694         if err != nil {
 695                 for k, v := range s.api.Logs {
 696                         c.Log(k)
 697                         c.Log(v.String())
 698                 }
 699         }
 700
 701         return s.api, s.runner, realTemp
 702 }
 703
 704 func (s *TestSuite) TestFullRunHello(c *C) {
 705         s.runner.enableMemoryLimit = true
 706         s.runner.networkMode = "default"
 707         s.fullRunHelper(c, `{
 708     "command": ["echo", "hello world"],
 709     "container_image": "`+arvadostest.DockerImage112PDH+`",
 710     "cwd": ".",
 711     "environment": {"foo":"bar","baz":"waz"},
 712     "mounts": {"/tmp": {"kind": "tmp"} },
 713     "output_path": "/tmp",
 714     "priority": 1,
 715     "runtime_constraints": {"vcpus":1,"ram":1000000},
 716     "state": "Locked",
 717     "output_storage_classes": ["default"]
 718 }`, nil, 0, func() {
 719                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 720                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 721                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 722                 c.Check(s.executor.created.VCPUs, Equals, 1)
 723                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 724                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 725                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 726                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 727                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 728         })
 729
 730         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 731         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 732         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 733         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 734         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 735 }
 736
 737 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 738         var ran bool
 739         s.fullRunHelper(c, `{
 740     "command": ["sleep", "3"],
 741     "container_image": "`+arvadostest.DockerImage112PDH+`",
 742     "cwd": ".",
 743     "environment": {},
 744     "mounts": {"/tmp": {"kind": "tmp"} },
 745     "output_path": "/tmp",
 746     "priority": 1,
 747     "runtime_constraints": {},
 748     "scheduling_parameters":{"max_run_time": 1},
 749     "state": "Running"
 750 }`, nil, 2, func() {
 751                 ran = true
 752         })
 753         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 754         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 755         c.Check(ran, Equals, false)
 756 }
 757
 758 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 759         s.fullRunHelper(c, `{
 760     "command": ["sleep", "3"],
 761     "container_image": "`+arvadostest.DockerImage112PDH+`",
 762     "cwd": ".",
 763     "environment": {},
 764     "mounts": {"/tmp": {"kind": "tmp"} },
 765     "output_path": "/tmp",
 766     "priority": 1,
 767     "runtime_constraints": {},
 768     "scheduling_parameters":{"max_run_time": 1},
 769     "state": "Locked"
 770 }`, nil, 0, func() {
 771                 time.Sleep(3 * time.Second)
 772         })
 773
 774         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 775         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 776 }
 777
 778 func (s *TestSuite) TestContainerWaitFails(c *C) {
 779         s.fullRunHelper(c, `{
 780     "command": ["sleep", "3"],
 781     "container_image": "`+arvadostest.DockerImage112PDH+`",
 782     "cwd": ".",
 783     "mounts": {"/tmp": {"kind": "tmp"} },
 784     "output_path": "/tmp",
 785     "priority": 1,
 786     "state": "Locked"
 787 }`, nil, 0, func() {
 788                 s.executor.waitErr = errors.New("Container is not running")
 789         })
 790
 791         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 792         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 793 }
 794
 795 func (s *TestSuite) TestCrunchstat(c *C) {
 796         s.fullRunHelper(c, `{
 797                 "command": ["sleep", "1"],
 798                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 799                 "cwd": ".",
 800                 "environment": {},
 801                 "mounts": {"/tmp": {"kind": "tmp"} },
 802                 "output_path": "/tmp",
 803                 "priority": 1,
 804                 "runtime_constraints": {},
 805                 "state": "Locked"
 806         }`, nil, 0, func() {
 807                 time.Sleep(time.Second)
 808         })
 809
 810         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 811         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 812
 813         // We didn't actually start a container, so crunchstat didn't
 814         // find accounting files and therefore didn't log any stats.
 815         // It should have logged a "can't find accounting files"
 816         // message after one poll interval, though, so we can confirm
 817         // it's alive:
 818         c.Assert(s.api.Logs["crunchstat"], NotNil)
 819         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 820
 821         // The "files never appeared" log assures us that we called
 822         // (*crunchstat.Reporter)Stop(), and that we set it up with
 823         // the correct container ID "abcde":
 824         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 825 }
 826
 827 func (s *TestSuite) TestNodeInfoLog(c *C) {
 828         os.Setenv("SLURMD_NODENAME", "compute2")
 829         s.fullRunHelper(c, `{
 830                 "command": ["sleep", "1"],
 831                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 832                 "cwd": ".",
 833                 "environment": {},
 834                 "mounts": {"/tmp": {"kind": "tmp"} },
 835                 "output_path": "/tmp",
 836                 "priority": 1,
 837                 "runtime_constraints": {},
 838                 "state": "Locked"
 839         }`, nil, 0,
 840                 func() {
 841                         time.Sleep(time.Second)
 842                 })
 843
 844         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 845         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 846
 847         c.Assert(s.api.Logs["node"], NotNil)
 848         json := s.api.Logs["node"].String()
 849         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 850         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 851         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 852
 853         c.Assert(s.api.Logs["node-info"], NotNil)
 854         json = s.api.Logs["node-info"].String()
 855         c.Check(json, Matches, `(?ms).*Host Information.*`)
 856         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 857         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 858         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 859         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 860 }
 861
 862 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 863         s.fullRunHelper(c, `{
 864                 "command": ["sleep", "1"],
 865                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 866                 "cwd": ".",
 867                 "environment": {},
 868                 "mounts": {"/tmp": {"kind": "tmp"} },
 869                 "output_path": "/tmp",
 870                 "priority": 1,
 871                 "runtime_constraints": {},
 872                 "state": "Locked"
 873         }`, nil, 0,
 874                 func() {
 875                 })
 876
 877         c.Assert(s.api.Logs["crunch-run"], NotNil)
 878         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 879         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 880 }
 881
 882 func (s *TestSuite) TestContainerRecordLog(c *C) {
 883         s.fullRunHelper(c, `{
 884                 "command": ["sleep", "1"],
 885                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 886                 "cwd": ".",
 887                 "environment": {},
 888                 "mounts": {"/tmp": {"kind": "tmp"} },
 889                 "output_path": "/tmp",
 890                 "priority": 1,
 891                 "runtime_constraints": {},
 892                 "state": "Locked"
 893         }`, nil, 0,
 894                 func() {
 895                         time.Sleep(time.Second)
 896                 })
 897
 898         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 899         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 900
 901         c.Assert(s.api.Logs["container"], NotNil)
 902         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 903 }
 904
 905 func (s *TestSuite) TestFullRunStderr(c *C) {
 906         s.fullRunHelper(c, `{
 907     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 908     "container_image": "`+arvadostest.DockerImage112PDH+`",
 909     "cwd": ".",
 910     "environment": {},
 911     "mounts": {"/tmp": {"kind": "tmp"} },
 912     "output_path": "/tmp",
 913     "priority": 1,
 914     "runtime_constraints": {},
 915     "state": "Locked"
 916 }`, nil, 1, func() {
 917                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 918                 fmt.Fprintln(s.executor.created.Stderr, "world")
 919         })
 920
 921         final := s.api.CalledWith("container.state", "Complete")
 922         c.Assert(final, NotNil)
 923         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 924         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 925
 926         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 927         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 928 }
 929
 930 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 931         s.fullRunHelper(c, `{
 932     "command": ["pwd"],
 933     "container_image": "`+arvadostest.DockerImage112PDH+`",
 934     "cwd": ".",
 935     "environment": {},
 936     "mounts": {"/tmp": {"kind": "tmp"} },
 937     "output_path": "/tmp",
 938     "priority": 1,
 939     "runtime_constraints": {},
 940     "state": "Locked"
 941 }`, nil, 0, func() {
 942                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 943         })
 944
 945         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 946         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 947         c.Log(s.api.Logs["stdout"])
 948         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 949 }
 950
 951 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 952         s.fullRunHelper(c, `{
 953     "command": ["pwd"],
 954     "container_image": "`+arvadostest.DockerImage112PDH+`",
 955     "cwd": "/bin",
 956     "environment": {},
 957     "mounts": {"/tmp": {"kind": "tmp"} },
 958     "output_path": "/tmp",
 959     "priority": 1,
 960     "runtime_constraints": {},
 961     "state": "Locked"
 962 }`, nil, 0, func() {
 963                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 964         })
 965
 966         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 967         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 968         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 969 }
 970
 971 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 972         s.fullRunHelper(c, `{
 973     "command": ["pwd"],
 974     "container_image": "`+arvadostest.DockerImage112PDH+`",
 975     "cwd": "/bin",
 976     "environment": {},
 977     "mounts": {"/tmp": {"kind": "tmp"} },
 978     "output_path": "/tmp",
 979     "priority": 1,
 980     "runtime_constraints": {},
 981     "state": "Locked",
 982     "output_storage_classes": ["foo", "bar"]
 983 }`, nil, 0, func() {
 984                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 985         })
 986
 987         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 988         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 989         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 990         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 991         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 992 }
 993
 994 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
 995         s.fullRunHelper(c, `{
 996     "command": ["pwd"],
 997     "container_image": "`+arvadostest.DockerImage112PDH+`",
 998     "cwd": "/bin",
 999     "environment": {},
1000     "mounts": {"/tmp": {"kind": "tmp"} },
1001     "output_path": "/tmp",
1002     "priority": 1,
1003     "runtime_constraints": {"cuda": {"device_count": 2}},
1004     "state": "Locked",
1005     "output_storage_classes": ["foo", "bar"]
1006 }`, nil, 0, func() {
1007                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1008         })
1009         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1010 }
1011
1012 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1013         s.fullRunHelper(c, `{
1014     "command": ["pwd"],
1015     "container_image": "`+arvadostest.DockerImage112PDH+`",
1016     "cwd": "/bin",
1017     "environment": {},
1018     "mounts": {"/tmp": {"kind": "tmp"} },
1019     "output_path": "/tmp",
1020     "priority": 1,
1021     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1022     "state": "Locked",
1023     "output_storage_classes": ["foo", "bar"]
1024 }`, nil, 0, func() {
1025                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1026         })
1027         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1028 }
1029
1030 func (s *TestSuite) TestStopOnSignal(c *C) {
1031         s.executor.runFunc = func() {
1032                 s.executor.created.Stdout.Write([]byte("foo\n"))
1033                 s.runner.SigChan <- syscall.SIGINT
1034         }
1035         s.testStopContainer(c)
1036 }
1037
1038 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1039         s.executor.runFunc = func() {
1040                 s.executor.created.Stdout.Write([]byte("foo\n"))
1041                 s.runner.ArvMountExit <- nil
1042                 close(s.runner.ArvMountExit)
1043         }
1044         s.runner.ArvMountExit = make(chan error)
1045         s.testStopContainer(c)
1046 }
1047
1048 func (s *TestSuite) testStopContainer(c *C) {
1049         record := `{
1050     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1051     "container_image": "` + arvadostest.DockerImage112PDH + `",
1052     "cwd": ".",
1053     "environment": {},
1054     "mounts": {"/tmp": {"kind": "tmp"} },
1055     "output_path": "/tmp",
1056     "priority": 1,
1057     "runtime_constraints": {},
1058     "state": "Locked"
1059 }`
1060
1061         err := json.Unmarshal([]byte(record), &s.api.Container)
1062         c.Assert(err, IsNil)
1063
1064         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1065         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1066                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1067         }
1068
1069         done := make(chan error)
1070         go func() {
1071                 done <- s.runner.Run()
1072         }()
1073         select {
1074         case <-time.After(20 * time.Second):
1075                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1076                 c.Fatal("timed out")
1077         case err = <-done:
1078                 c.Check(err, IsNil)
1079         }
1080         for k, v := range s.api.Logs {
1081                 c.Log(k)
1082                 c.Log(v.String(), "\n")
1083         }
1084
1085         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1086         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1087         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1088 }
1089
1090 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1091         s.fullRunHelper(c, `{
1092     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1093     "container_image": "`+arvadostest.DockerImage112PDH+`",
1094     "cwd": "/bin",
1095     "environment": {"FROBIZ": "bilbo"},
1096     "mounts": {"/tmp": {"kind": "tmp"} },
1097     "output_path": "/tmp",
1098     "priority": 1,
1099     "runtime_constraints": {},
1100     "state": "Locked"
1101 }`, nil, 0, func() {
1102                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1103         })
1104
1105         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1106         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1107         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1108 }
1109
1110 type ArvMountCmdLine struct {
1111         Cmd   []string
1112         token string
1113 }
1114
1115 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1116         am.Cmd = c
1117         am.token = token
1118         return nil, nil
1119 }
1120
1121 func stubCert(temp string) string {
1122         path := temp + "/ca-certificates.crt"
1123         crt, _ := os.Create(path)
1124         crt.Close()
1125         arvadosclient.CertFiles = []string{path}
1126         return path
1127 }
1128
1129 func (s *TestSuite) TestSetupMounts(c *C) {
1130         cr := s.runner
1131         am := &ArvMountCmdLine{}
1132         cr.RunArvMount = am.ArvMountTest
1133         cr.ContainerArvClient = &ArvTestClient{}
1134         cr.ContainerKeepClient = &KeepTestClient{}
1135         cr.Container.OutputStorageClasses = []string{"default"}
1136
1137         realTemp := c.MkDir()
1138         certTemp := c.MkDir()
1139         stubCertPath := stubCert(certTemp)
1140         cr.parentTemp = realTemp
1141
1142         i := 0
1143         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1144                 i++
1145                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1146                 err := os.Mkdir(d, os.ModePerm)
1147                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1148                         // Test case must have pre-populated the tempdir
1149                         err = nil
1150                 }
1151                 return d, err
1152         }
1153
1154         checkEmpty := func() {
1155                 // Should be deleted.
1156                 _, err := os.Stat(realTemp)
1157                 c.Assert(os.IsNotExist(err), Equals, true)
1158
1159                 // Now recreate it for the next test.
1160                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1161         }
1162
1163         {
1164                 i = 0
1165                 cr.ArvMountPoint = ""
1166                 cr.Container.Mounts = make(map[string]arvados.Mount)
1167                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1168                 cr.Container.OutputPath = "/tmp"
1169                 cr.statInterval = 5 * time.Second
1170                 bindmounts, err := cr.SetupMounts()
1171                 c.Check(err, IsNil)
1172                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1173                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1174                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1175                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1176                 os.RemoveAll(cr.ArvMountPoint)
1177                 cr.CleanupDirs()
1178                 checkEmpty()
1179         }
1180
1181         {
1182                 i = 0
1183                 cr.ArvMountPoint = ""
1184                 cr.Container.Mounts = make(map[string]arvados.Mount)
1185                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1186                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1187                 cr.Container.OutputPath = "/out"
1188                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1189
1190                 bindmounts, err := cr.SetupMounts()
1191                 c.Check(err, IsNil)
1192                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1193                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1194                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1195                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1196                 os.RemoveAll(cr.ArvMountPoint)
1197                 cr.CleanupDirs()
1198                 checkEmpty()
1199         }
1200
1201         {
1202                 i = 0
1203                 cr.ArvMountPoint = ""
1204                 cr.Container.Mounts = make(map[string]arvados.Mount)
1205                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1206                 cr.Container.OutputPath = "/tmp"
1207                 cr.Container.RuntimeConstraints.API = true
1208                 cr.Container.OutputStorageClasses = []string{"default"}
1209
1210                 bindmounts, err := cr.SetupMounts()
1211                 c.Check(err, IsNil)
1212                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1213                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1214                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1215                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1216                 os.RemoveAll(cr.ArvMountPoint)
1217                 cr.CleanupDirs()
1218                 checkEmpty()
1219
1220                 cr.Container.RuntimeConstraints.API = false
1221         }
1222
1223         {
1224                 i = 0
1225                 cr.ArvMountPoint = ""
1226                 cr.Container.Mounts = map[string]arvados.Mount{
1227                         "/keeptmp": {Kind: "collection", Writable: true},
1228                 }
1229                 cr.Container.OutputPath = "/keeptmp"
1230
1231                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1232
1233                 bindmounts, err := cr.SetupMounts()
1234                 c.Check(err, IsNil)
1235                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1236                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1237                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1238                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1239                 os.RemoveAll(cr.ArvMountPoint)
1240                 cr.CleanupDirs()
1241                 checkEmpty()
1242         }
1243
1244         {
1245                 i = 0
1246                 cr.ArvMountPoint = ""
1247                 cr.Container.Mounts = map[string]arvados.Mount{
1248                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1249                         "/keepout": {Kind: "collection", Writable: true},
1250                 }
1251                 cr.Container.OutputPath = "/keepout"
1252
1253                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1254                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1255
1256                 bindmounts, err := cr.SetupMounts()
1257                 c.Check(err, IsNil)
1258                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1259                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1260                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1261                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1262                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1263                         "/keepout": {realTemp + "/keep1/tmp0", false},
1264                 })
1265                 os.RemoveAll(cr.ArvMountPoint)
1266                 cr.CleanupDirs()
1267                 checkEmpty()
1268         }
1269
1270         {
1271                 i = 0
1272                 cr.ArvMountPoint = ""
1273                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1274                 cr.Container.Mounts = map[string]arvados.Mount{
1275                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1276                         "/keepout": {Kind: "collection", Writable: true},
1277                 }
1278                 cr.Container.OutputPath = "/keepout"
1279
1280                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1281                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1282
1283                 bindmounts, err := cr.SetupMounts()
1284                 c.Check(err, IsNil)
1285                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1286                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1287                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1288                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1289                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1290                         "/keepout": {realTemp + "/keep1/tmp0", false},
1291                 })
1292                 os.RemoveAll(cr.ArvMountPoint)
1293                 cr.CleanupDirs()
1294                 checkEmpty()
1295         }
1296
1297         for _, test := range []struct {
1298                 in  interface{}
1299                 out string
1300         }{
1301                 {in: "foo", out: `"foo"`},
1302                 {in: nil, out: `null`},
1303                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1304         } {
1305                 i = 0
1306                 cr.ArvMountPoint = ""
1307                 cr.Container.Mounts = map[string]arvados.Mount{
1308                         "/mnt/test.json": {Kind: "json", Content: test.in},
1309                 }
1310                 bindmounts, err := cr.SetupMounts()
1311                 c.Check(err, IsNil)
1312                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1313                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1314                 })
1315                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1316                 c.Check(err, IsNil)
1317                 c.Check(content, DeepEquals, []byte(test.out))
1318                 os.RemoveAll(cr.ArvMountPoint)
1319                 cr.CleanupDirs()
1320                 checkEmpty()
1321         }
1322
1323         for _, test := range []struct {
1324                 in  interface{}
1325                 out string
1326         }{
1327                 {in: "foo", out: `foo`},
1328                 {in: nil, out: "error"},
1329                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1330         } {
1331                 i = 0
1332                 cr.ArvMountPoint = ""
1333                 cr.Container.Mounts = map[string]arvados.Mount{
1334                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1335                 }
1336                 bindmounts, err := cr.SetupMounts()
1337                 if test.out == "error" {
1338                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1339                 } else {
1340                         c.Check(err, IsNil)
1341                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1342                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1343                         })
1344                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1345                         c.Check(err, IsNil)
1346                         c.Check(content, DeepEquals, []byte(test.out))
1347                 }
1348                 os.RemoveAll(cr.ArvMountPoint)
1349                 cr.CleanupDirs()
1350                 checkEmpty()
1351         }
1352
1353         // Read-only mount points are allowed underneath output_dir mount point
1354         {
1355                 i = 0
1356                 cr.ArvMountPoint = ""
1357                 cr.Container.Mounts = make(map[string]arvados.Mount)
1358                 cr.Container.Mounts = map[string]arvados.Mount{
1359                         "/tmp":     {Kind: "tmp"},
1360                         "/tmp/foo": {Kind: "collection"},
1361                 }
1362                 cr.Container.OutputPath = "/tmp"
1363
1364                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1365
1366                 bindmounts, err := cr.SetupMounts()
1367                 c.Check(err, IsNil)
1368                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1369                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1370                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1371                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1372                         "/tmp":     {realTemp + "/tmp2", false},
1373                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1374                 })
1375                 os.RemoveAll(cr.ArvMountPoint)
1376                 cr.CleanupDirs()
1377                 checkEmpty()
1378         }
1379
1380         // Writable mount points copied to output_dir mount point
1381         {
1382                 i = 0
1383                 cr.ArvMountPoint = ""
1384                 cr.Container.Mounts = make(map[string]arvados.Mount)
1385                 cr.Container.Mounts = map[string]arvados.Mount{
1386                         "/tmp": {Kind: "tmp"},
1387                         "/tmp/foo": {Kind: "collection",
1388                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1389                                 Writable:         true},
1390                         "/tmp/bar": {Kind: "collection",
1391                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1392                                 Path:             "baz",
1393                                 Writable:         true},
1394                 }
1395                 cr.Container.OutputPath = "/tmp"
1396
1397                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1398                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1399
1400                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1401                 rf.Write([]byte("bar"))
1402                 rf.Close()
1403
1404                 _, err := cr.SetupMounts()
1405                 c.Check(err, IsNil)
1406                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1407                 c.Check(err, IsNil)
1408                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1409                 c.Check(err, IsNil)
1410                 os.RemoveAll(cr.ArvMountPoint)
1411                 cr.CleanupDirs()
1412                 checkEmpty()
1413         }
1414
1415         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1416         {
1417                 i = 0
1418                 cr.ArvMountPoint = ""
1419                 cr.Container.Mounts = make(map[string]arvados.Mount)
1420                 cr.Container.Mounts = map[string]arvados.Mount{
1421                         "/tmp":     {Kind: "tmp"},
1422                         "/tmp/foo": {Kind: "tmp"},
1423                 }
1424                 cr.Container.OutputPath = "/tmp"
1425
1426                 _, err := cr.SetupMounts()
1427                 c.Check(err, NotNil)
1428                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1429                 os.RemoveAll(cr.ArvMountPoint)
1430                 cr.CleanupDirs()
1431                 checkEmpty()
1432         }
1433
1434         // Only mount point of kind 'collection' is allowed for stdin
1435         {
1436                 i = 0
1437                 cr.ArvMountPoint = ""
1438                 cr.Container.Mounts = make(map[string]arvados.Mount)
1439                 cr.Container.Mounts = map[string]arvados.Mount{
1440                         "stdin": {Kind: "tmp"},
1441                 }
1442
1443                 _, err := cr.SetupMounts()
1444                 c.Check(err, NotNil)
1445                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1446                 os.RemoveAll(cr.ArvMountPoint)
1447                 cr.CleanupDirs()
1448                 checkEmpty()
1449         }
1450
1451         // git_tree mounts
1452         {
1453                 i = 0
1454                 cr.ArvMountPoint = ""
1455                 (*GitMountSuite)(nil).useTestGitServer(c)
1456                 cr.token = arvadostest.ActiveToken
1457                 cr.Container.Mounts = make(map[string]arvados.Mount)
1458                 cr.Container.Mounts = map[string]arvados.Mount{
1459                         "/tip": {
1460                                 Kind:   "git_tree",
1461                                 UUID:   arvadostest.Repository2UUID,
1462                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1463                                 Path:   "/",
1464                         },
1465                         "/non-tip": {
1466                                 Kind:   "git_tree",
1467                                 UUID:   arvadostest.Repository2UUID,
1468                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1469                                 Path:   "/",
1470                         },
1471                 }
1472                 cr.Container.OutputPath = "/tmp"
1473
1474                 bindmounts, err := cr.SetupMounts()
1475                 c.Check(err, IsNil)
1476
1477                 for path, mount := range bindmounts {
1478                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1479                 }
1480
1481                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1482                 c.Check(err, IsNil)
1483                 c.Check(string(data), Equals, "\000\001\002\003")
1484                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1485                 c.Check(err, FitsTypeOf, &os.PathError{})
1486                 c.Check(os.IsNotExist(err), Equals, true)
1487
1488                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1489                 c.Check(err, IsNil)
1490                 c.Check(string(data), Equals, "\000\001\002\003")
1491                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1492                 c.Check(err, IsNil)
1493                 c.Check(string(data), Equals, "testfile\n")
1494
1495                 cr.CleanupDirs()
1496                 checkEmpty()
1497         }
1498 }
1499
1500 func (s *TestSuite) TestStdout(c *C) {
1501         helperRecord := `{
1502                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1503                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1504                 "cwd": "/bin",
1505                 "environment": {"FROBIZ": "bilbo"},
1506                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1507                 "output_path": "/tmp",
1508                 "priority": 1,
1509                 "runtime_constraints": {},
1510                 "state": "Locked"
1511         }`
1512
1513         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1514                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1515         })
1516
1517         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1518         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1519         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1520 }
1521
1522 // Used by the TestStdoutWithWrongPath*()
1523 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1524         err := json.Unmarshal([]byte(record), &s.api.Container)
1525         c.Assert(err, IsNil)
1526         s.executor.runFunc = fn
1527         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1528         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1529                 return s.api, &KeepTestClient{}, nil, nil
1530         }
1531         return s.api, s.runner, s.runner.Run()
1532 }
1533
1534 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1535         _, _, err := s.stdoutErrorRunHelper(c, `{
1536     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1537     "output_path": "/tmp",
1538     "state": "Locked"
1539 }`, func() {})
1540         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1541 }
1542
1543 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1544         _, _, err := s.stdoutErrorRunHelper(c, `{
1545     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1546     "output_path": "/tmp",
1547     "state": "Locked"
1548 }`, func() {})
1549         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1550 }
1551
1552 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1553         _, _, err := s.stdoutErrorRunHelper(c, `{
1554     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1555     "output_path": "/tmp",
1556     "state": "Locked"
1557 }`, func() {})
1558         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1559 }
1560
1561 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1562         s.fullRunHelper(c, `{
1563     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1564     "container_image": "`+arvadostest.DockerImage112PDH+`",
1565     "cwd": "/bin",
1566     "environment": {},
1567     "mounts": {"/tmp": {"kind": "tmp"} },
1568     "output_path": "/tmp",
1569     "priority": 1,
1570     "runtime_constraints": {"API": true},
1571     "state": "Locked"
1572 }`, nil, 0, func() {
1573                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1574                 s.executor.exit <- 3
1575         })
1576         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1577         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1578         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1579 }
1580
1581 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1582         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1583         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1584         s.fullRunHelper(c, `{
1585     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1586     "container_image": "`+arvadostest.DockerImage112PDH+`",
1587     "cwd": "/bin",
1588     "environment": {},
1589     "mounts": {"/tmp": {"kind": "tmp"} },
1590     "output_path": "/tmp",
1591     "priority": 1,
1592     "runtime_constraints": {"API": true},
1593     "state": "Locked"
1594 }`, nil, 0, func() {
1595                 s.api.Container.Output = arvadostest.DockerImage112PDH
1596         })
1597
1598         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1599         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1600         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1601 }
1602
1603 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1604         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1605                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1606                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1607                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1608         }
1609         s.executor.runFunc = func() {
1610                 time.Sleep(time.Second)
1611                 s.executor.exit <- 137
1612         }
1613         record := `{
1614     "command": ["sleep", "1"],
1615     "container_image": "` + arvadostest.DockerImage112PDH + `",
1616     "cwd": "/bin",
1617     "environment": {},
1618     "mounts": {"/tmp": {"kind": "tmp"} },
1619     "output_path": "/tmp",
1620     "priority": 1,
1621     "runtime_constraints": {"API": true},
1622     "state": "Locked"
1623 }`
1624         err := json.Unmarshal([]byte(record), &s.api.Container)
1625         c.Assert(err, IsNil)
1626         err = s.runner.Run()
1627         c.Assert(err, IsNil)
1628         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1629         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1630         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1631         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1632         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1633 }
1634
1635 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1636         helperRecord := `{
1637                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1638                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1639                 "cwd": "/bin",
1640                 "environment": {"FROBIZ": "bilbo"},
1641                 "mounts": {
1642         "/tmp": {"kind": "tmp"},
1643         "/tmp/foo": {"kind": "collection",
1644                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1645                      "exclude_from_output": true
1646         },
1647         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1648     },
1649                 "output_path": "/tmp",
1650                 "priority": 1,
1651                 "runtime_constraints": {},
1652                 "state": "Locked"
1653         }`
1654
1655         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1656
1657         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1658                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1659         })
1660
1661         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1662         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1663         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1664 }
1665
1666 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1667         helperRecord := `{
1668                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1669                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1670                 "cwd": "/bin",
1671                 "environment": {"FROBIZ": "bilbo"},
1672                 "mounts": {
1673         "/tmp": {"kind": "tmp"},
1674         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1675         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1676         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1677         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1678         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1679     },
1680                 "output_path": "/tmp",
1681                 "priority": 1,
1682                 "runtime_constraints": {},
1683                 "state": "Locked"
1684         }`
1685
1686         extraMounts := []string{
1687                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1688                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1689                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1690         }
1691
1692         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1693                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1694         })
1695
1696         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1697                 "/tmp":                   {realtemp + "/tmp1", false},
1698                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1699                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1700                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1701                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1702         })
1703
1704         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1705         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1706         for _, v := range api.Content {
1707                 if v["collection"] != nil {
1708                         c.Check(v["ensure_unique_name"], Equals, true)
1709                         collection := v["collection"].(arvadosclient.Dict)
1710                         if strings.Index(collection["name"].(string), "output") == 0 {
1711                                 manifest := collection["manifest_text"].(string)
1712
1713                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1714 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1715 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1716 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1717 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1718 `)
1719                         }
1720                 }
1721         }
1722 }
1723
1724 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1725         helperRecord := `{
1726                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1727                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1728                 "cwd": "/bin",
1729                 "environment": {"FROBIZ": "bilbo"},
1730                 "mounts": {
1731         "/tmp": {"kind": "tmp"},
1732         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1733         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1734     },
1735                 "output_path": "/tmp",
1736                 "priority": 1,
1737                 "runtime_constraints": {},
1738                 "state": "Locked"
1739         }`
1740
1741         extraMounts := []string{
1742                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1743         }
1744
1745         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1746                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1747         })
1748
1749         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1750         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1751         for _, v := range s.api.Content {
1752                 if v["collection"] != nil {
1753                         collection := v["collection"].(arvadosclient.Dict)
1754                         if strings.Index(collection["name"].(string), "output") == 0 {
1755                                 manifest := collection["manifest_text"].(string)
1756
1757                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1758 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1759 `)
1760                         }
1761                 }
1762         }
1763 }
1764
1765 func (s *TestSuite) TestOutputError(c *C) {
1766         helperRecord := `{
1767                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1768                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1769                 "cwd": "/bin",
1770                 "environment": {"FROBIZ": "bilbo"},
1771                 "mounts": {
1772                         "/tmp": {"kind": "tmp"}
1773                 },
1774                 "output_path": "/tmp",
1775                 "priority": 1,
1776                 "runtime_constraints": {},
1777                 "state": "Locked"
1778         }`
1779         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1780                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1781         })
1782
1783         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1784 }
1785
1786 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1787         helperRecord := `{
1788                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1789                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1790                 "cwd": "/bin",
1791                 "environment": {"FROBIZ": "bilbo"},
1792                 "mounts": {
1793         "/tmp": {"kind": "tmp"},
1794         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1795         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1796     },
1797                 "output_path": "/tmp",
1798                 "priority": 1,
1799                 "runtime_constraints": {},
1800                 "state": "Locked"
1801         }`
1802
1803         extraMounts := []string{
1804                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1805         }
1806
1807         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1808                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1809         })
1810
1811         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1812         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1813         for _, v := range api.Content {
1814                 if v["collection"] != nil {
1815                         collection := v["collection"].(arvadosclient.Dict)
1816                         if strings.Index(collection["name"].(string), "output") == 0 {
1817                                 manifest := collection["manifest_text"].(string)
1818                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1819 `)
1820                         }
1821                 }
1822         }
1823 }
1824
1825 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1826         helperRecord := `{
1827                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1828                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1829                 "cwd": "/bin",
1830                 "environment": {"FROBIZ": "bilbo"},
1831                 "mounts": {
1832         "/tmp": {"kind": "tmp"},
1833         "stdin": {"kind": "json", "content": "foo"},
1834         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1835     },
1836                 "output_path": "/tmp",
1837                 "priority": 1,
1838                 "runtime_constraints": {},
1839                 "state": "Locked"
1840         }`
1841
1842         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1843                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1844         })
1845
1846         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1847         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1848         for _, v := range api.Content {
1849                 if v["collection"] != nil {
1850                         collection := v["collection"].(arvadosclient.Dict)
1851                         if strings.Index(collection["name"].(string), "output") == 0 {
1852                                 manifest := collection["manifest_text"].(string)
1853                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1854 `)
1855                         }
1856                 }
1857         }
1858 }
1859
1860 func (s *TestSuite) TestStderrMount(c *C) {
1861         api, cr, _ := s.fullRunHelper(c, `{
1862     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1863     "container_image": "`+arvadostest.DockerImage112PDH+`",
1864     "cwd": ".",
1865     "environment": {},
1866     "mounts": {"/tmp": {"kind": "tmp"},
1867                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1868                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1869     "output_path": "/tmp",
1870     "priority": 1,
1871     "runtime_constraints": {},
1872     "state": "Locked"
1873 }`, nil, 1, func() {
1874                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1875                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1876         })
1877
1878         final := api.CalledWith("container.state", "Complete")
1879         c.Assert(final, NotNil)
1880         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1881         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1882
1883         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1884 }
1885
1886 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1887         s.api.callraw = true
1888         err := s.runner.fetchContainerRecord()
1889         c.Assert(err, IsNil)
1890         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1891         c.Logf("%#v", s.runner.Container)
1892         c.Check(err, IsNil)
1893         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1894 }
1895
1896 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1897         nextState := ""
1898         for _, setup := range []func(){
1899                 func() {
1900                         c.Log("// waitErr = ocl runtime error")
1901                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1902                         nextState = "Cancelled"
1903                 },
1904                 func() {
1905                         c.Log("// loadErr = cannot connect")
1906                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1907                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1908                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1909                         c.Assert(err, IsNil)
1910                         nextState = "Queued"
1911                 },
1912         } {
1913                 s.SetUpTest(c)
1914                 setup()
1915                 s.fullRunHelper(c, `{
1916     "command": ["echo", "hello world"],
1917     "container_image": "`+arvadostest.DockerImage112PDH+`",
1918     "cwd": ".",
1919     "environment": {},
1920     "mounts": {"/tmp": {"kind": "tmp"} },
1921     "output_path": "/tmp",
1922     "priority": 1,
1923     "runtime_constraints": {},
1924     "state": "Locked"
1925 }`, nil, 0, func() {})
1926                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1927                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1928                 if *brokenNodeHook != "" {
1929                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1930                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1931                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1932                 } else {
1933                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1934                 }
1935         }
1936 }
1937
1938 func (s *TestSuite) TestBadCommand(c *C) {
1939         for _, startError := range []string{
1940                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1941                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1942                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1943         } {
1944                 s.SetUpTest(c)
1945                 s.executor.startErr = errors.New(startError)
1946                 s.fullRunHelper(c, `{
1947     "command": ["echo", "hello world"],
1948     "container_image": "`+arvadostest.DockerImage112PDH+`",
1949     "cwd": ".",
1950     "environment": {},
1951     "mounts": {"/tmp": {"kind": "tmp"} },
1952     "output_path": "/tmp",
1953     "priority": 1,
1954     "runtime_constraints": {},
1955     "state": "Locked"
1956 }`, nil, 0, func() {})
1957                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1958                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1959         }
1960 }
1961
1962 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1963         helperRecord := `{
1964                 "command": ["true"],
1965                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1966                 "cwd": "/bin",
1967                 "mounts": {
1968                     "/tmp": {"kind": "tmp"},
1969                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1970                 },
1971                 "secret_mounts": {
1972                 },
1973                 "output_path": "/tmp",
1974                 "priority": 1,
1975                 "runtime_constraints": {},
1976                 "state": "Locked"
1977         }`
1978
1979         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1980                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1981                 c.Check(err, IsNil)
1982                 c.Check(string(content), Equals, "mypassword")
1983         })
1984
1985         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1986         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1987         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1988         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1989
1990         // under secret mounts, not captured in output
1991         helperRecord = `{
1992                 "command": ["true"],
1993                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1994                 "cwd": "/bin",
1995                 "mounts": {
1996                     "/tmp": {"kind": "tmp"}
1997                 },
1998                 "secret_mounts": {
1999                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2000                 },
2001                 "output_path": "/tmp",
2002                 "priority": 1,
2003                 "runtime_constraints": {},
2004                 "state": "Locked"
2005         }`
2006
2007         s.SetUpTest(c)
2008         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2009                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2010                 c.Check(err, IsNil)
2011                 c.Check(string(content), Equals, "mypassword")
2012         })
2013
2014         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2015         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2016         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2017         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2018 }
2019
2020 type FakeProcess struct {
2021         cmdLine []string
2022 }
2023
2024 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2025         return fp.cmdLine, nil
2026 }