lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 116         containerClient *arvados.Client) error {
 117         e.loaded = tarball
 118         return e.loadErr
 119 }
 120 func (e *stubExecutor) Runtime() string                 { return "stub" }
 121 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 122 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 123 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 124 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 125 func (e *stubExecutor) Close()                          { e.closed = true }
 126 func (e *stubExecutor) Wait(context.Context) (int, error) {
 127         return <-e.exit, e.waitErr
 128 }
 129
 130 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 131
 132 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 133 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 134 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 135
 136 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 137 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 138
 139 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 140 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 141 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 142 `
 143
 144 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 145
 146 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 147 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 148
 149 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 150 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 151
 152 func (client *ArvTestClient) Create(resourceType string,
 153         parameters arvadosclient.Dict,
 154         output interface{}) error {
 155
 156         client.Mutex.Lock()
 157         defer client.Mutex.Unlock()
 158
 159         client.Calls++
 160         client.Content = append(client.Content, parameters)
 161
 162         if resourceType == "logs" {
 163                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 164                 if client.Logs == nil {
 165                         client.Logs = make(map[string]*bytes.Buffer)
 166                 }
 167                 if client.Logs[et] == nil {
 168                         client.Logs[et] = &bytes.Buffer{}
 169                 }
 170                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 171         }
 172
 173         if resourceType == "collections" && output != nil {
 174                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 175                 outmap := output.(*arvados.Collection)
 176                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 177                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 178         }
 179
 180         return nil
 181 }
 182
 183 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 184         switch {
 185         case method == "GET" && resourceType == "containers" && action == "auth":
 186                 return json.Unmarshal([]byte(`{
 187                         "kind": "arvados#api_client_authorization",
 188                         "uuid": "`+fakeAuthUUID+`",
 189                         "api_token": "`+fakeAuthToken+`"
 190                         }`), output)
 191         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 192                 if client.secretMounts != nil {
 193                         return json.Unmarshal(client.secretMounts, output)
 194                 }
 195                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 196         default:
 197                 return fmt.Errorf("Not found")
 198         }
 199 }
 200
 201 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 202         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 203         var j []byte
 204         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 205                 j = []byte(`{
 206                         "kind": "arvados#nodeList",
 207                         "items": [{
 208                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 209                                 "hostname": "compute2",
 210                                 "properties": {"total_cpu_cores": 16}
 211                         }]}`)
 212         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 213                 if uuid == "" {
 214                         j, err = json.Marshal(map[string]interface{}{
 215                                 "items": []interface{}{client.Container},
 216                                 "kind":  "arvados#nodeList",
 217                         })
 218                 } else {
 219                         j, err = json.Marshal(client.Container)
 220                 }
 221         } else {
 222                 j = []byte(`{
 223                         "command": ["sleep", "1"],
 224                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 225                         "cwd": ".",
 226                         "environment": {},
 227                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 228                         "output_path": "/tmp",
 229                         "priority": 1,
 230                         "runtime_constraints": {}
 231                 }`)
 232         }
 233         return ioutil.NopCloser(bytes.NewReader(j)), err
 234 }
 235
 236 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 237         if resourceType == "collections" {
 238                 if uuid == hwPDH {
 239                         output.(*arvados.Collection).ManifestText = hwManifest
 240                 } else if uuid == otherPDH {
 241                         output.(*arvados.Collection).ManifestText = otherManifest
 242                 } else if uuid == normalizedWithSubdirsPDH {
 243                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 244                 } else if uuid == denormalizedWithSubdirsPDH {
 245                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 246                 }
 247         }
 248         if resourceType == "containers" {
 249                 (*output.(*arvados.Container)) = client.Container
 250         }
 251         return nil
 252 }
 253
 254 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 255         client.Mutex.Lock()
 256         defer client.Mutex.Unlock()
 257         client.Calls++
 258         client.Content = append(client.Content, parameters)
 259         if resourceType == "containers" {
 260                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 261                         client.WasSetRunning = true
 262                 }
 263         } else if resourceType == "collections" {
 264                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 265                 output.(*arvados.Collection).UUID = uuid
 266                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 267         }
 268         return nil
 269 }
 270
 271 var discoveryMap = map[string]interface{}{
 272         "defaultTrashLifetime":               float64(1209600),
 273         "crunchLimitLogBytesPerJob":          float64(67108864),
 274         "crunchLogThrottleBytes":             float64(65536),
 275         "crunchLogThrottlePeriod":            float64(60),
 276         "crunchLogThrottleLines":             float64(1024),
 277         "crunchLogPartialLineThrottlePeriod": float64(5),
 278         "crunchLogBytesPerEvent":             float64(4096),
 279         "crunchLogSecondsBetweenEvents":      float64(1),
 280 }
 281
 282 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 283         return discoveryMap[key], nil
 284 }
 285
 286 // CalledWith returns the parameters from the first API call whose
 287 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 288 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 289 // no call matches, it returns nil.
 290 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 291 call:
 292         for _, content := range client.Content {
 293                 var v interface{} = content
 294                 for _, k := range strings.Split(jpath, ".") {
 295                         if dict, ok := v.(arvadosclient.Dict); !ok {
 296                                 continue call
 297                         } else {
 298                                 v = dict[k]
 299                         }
 300                 }
 301                 if v == expect {
 302                         return content
 303                 }
 304         }
 305         return nil
 306 }
 307
 308 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 309         return locator, nil
 310 }
 311
 312 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 313         client.Content = opts.Data
 314         return arvados.BlockWriteResponse{
 315                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 316         }, nil
 317 }
 318
 319 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 320         return 0, errors.New("not implemented")
 321 }
 322
 323 func (client *KeepTestClient) ClearBlockCache() {
 324 }
 325
 326 func (client *KeepTestClient) Close() {
 327         client.Content = nil
 328 }
 329
 330 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 331         client.StorageClasses = sc
 332 }
 333
 334 type FileWrapper struct {
 335         io.ReadCloser
 336         len int64
 337 }
 338
 339 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 340         return nil, errors.New("not implemented")
 341 }
 342
 343 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 344         return 0, errors.New("not implemented")
 345 }
 346
 347 func (fw FileWrapper) Size() int64 {
 348         return fw.len
 349 }
 350
 351 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 352         return nil, errors.New("not implemented")
 353 }
 354
 355 func (fw FileWrapper) Truncate(int64) error {
 356         return errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Write([]byte) (int, error) {
 360         return 0, errors.New("not implemented")
 361 }
 362
 363 func (fw FileWrapper) Sync() error {
 364         return errors.New("not implemented")
 365 }
 366
 367 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 368         if filename == hwImageID+".tar" {
 369                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 370                 client.Called = true
 371                 return FileWrapper{rdr, 1321984}, nil
 372         } else if filename == "/file1_in_main.txt" {
 373                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 374                 client.Called = true
 375                 return FileWrapper{rdr, 3}, nil
 376         }
 377         return nil, nil
 378 }
 379
 380 func (s *TestSuite) TestLoadImage(c *C) {
 381         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 382         s.runner.Container.Mounts = map[string]arvados.Mount{
 383                 "/out": {Kind: "tmp", Writable: true},
 384         }
 385         s.runner.Container.OutputPath = "/out"
 386
 387         _, err := s.runner.SetupMounts()
 388         c.Assert(err, IsNil)
 389
 390         imageID, err := s.runner.LoadImage()
 391         c.Check(err, IsNil)
 392         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 393         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 394
 395         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 396         s.executor.imageLoaded = false
 397         s.executor.loaded = ""
 398         s.executor.loadErr = errors.New("bork")
 399         imageID, err = s.runner.LoadImage()
 400         c.Check(err, ErrorMatches, ".*bork")
 401         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 402
 403         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 404         s.executor.imageLoaded = false
 405         s.executor.loaded = ""
 406         s.executor.loadErr = nil
 407         imageID, err = s.runner.LoadImage()
 408         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 409         c.Check(s.executor.loaded, Equals, "")
 410 }
 411
 412 type ArvErrorTestClient struct{}
 413
 414 func (ArvErrorTestClient) Create(resourceType string,
 415         parameters arvadosclient.Dict,
 416         output interface{}) error {
 417         return nil
 418 }
 419
 420 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 421         if method == "GET" && resourceType == "containers" && action == "auth" {
 422                 return nil
 423         }
 424         return errors.New("ArvError")
 425 }
 426
 427 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 428         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 429         return nil, errors.New("ArvError")
 430 }
 431
 432 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 433         return errors.New("ArvError")
 434 }
 435
 436 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 437         return nil
 438 }
 439
 440 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 441         return discoveryMap[key], nil
 442 }
 443
 444 type KeepErrorTestClient struct {
 445         KeepTestClient
 446 }
 447
 448 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 449         return nil, errors.New("KeepError")
 450 }
 451
 452 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 453         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 454 }
 455
 456 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 457         return "", errors.New("KeepError")
 458 }
 459
 460 type KeepReadErrorTestClient struct {
 461         KeepTestClient
 462 }
 463
 464 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 465         return 0, errors.New("KeepError")
 466 }
 467
 468 type ErrorReader struct {
 469         FileWrapper
 470 }
 471
 472 func (ErrorReader) Read(p []byte) (n int, err error) {
 473         return 0, errors.New("ErrorReader")
 474 }
 475
 476 func (ErrorReader) Seek(int64, int) (int64, error) {
 477         return 0, errors.New("ErrorReader")
 478 }
 479
 480 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 481         return ErrorReader{}, nil
 482 }
 483
 484 type ClosableBuffer struct {
 485         bytes.Buffer
 486 }
 487
 488 func (*ClosableBuffer) Close() error {
 489         return nil
 490 }
 491
 492 type TestLogs struct {
 493         Stdout ClosableBuffer
 494         Stderr ClosableBuffer
 495 }
 496
 497 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 498         if logstr == "stdout" {
 499                 return &tl.Stdout, nil
 500         }
 501         if logstr == "stderr" {
 502                 return &tl.Stderr, nil
 503         }
 504         return nil, errors.New("???")
 505 }
 506
 507 func dockerLog(fd byte, msg string) []byte {
 508         by := []byte(msg)
 509         header := make([]byte, 8+len(by))
 510         header[0] = fd
 511         header[7] = byte(len(by))
 512         copy(header[8:], by)
 513         return header
 514 }
 515
 516 func (s *TestSuite) TestRunContainer(c *C) {
 517         s.executor.runFunc = func() {
 518                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 519                 s.executor.exit <- 0
 520         }
 521
 522         var logs TestLogs
 523         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 524         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 525         s.runner.Container.Command = []string{"./hw"}
 526         s.runner.Container.OutputStorageClasses = []string{"default"}
 527
 528         imageID, err := s.runner.LoadImage()
 529         c.Assert(err, IsNil)
 530
 531         err = s.runner.CreateContainer(imageID, nil)
 532         c.Assert(err, IsNil)
 533
 534         err = s.runner.StartContainer()
 535         c.Assert(err, IsNil)
 536
 537         err = s.runner.WaitFinish()
 538         c.Assert(err, IsNil)
 539
 540         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 541         c.Check(logs.Stderr.String(), Equals, "")
 542 }
 543
 544 func (s *TestSuite) TestCommitLogs(c *C) {
 545         api := &ArvTestClient{}
 546         kc := &KeepTestClient{}
 547         defer kc.Close()
 548         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 549         c.Assert(err, IsNil)
 550         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 551
 552         cr.CrunchLog.Print("Hello world!")
 553         cr.CrunchLog.Print("Goodbye")
 554         cr.finalState = "Complete"
 555
 556         err = cr.CommitLogs()
 557         c.Check(err, IsNil)
 558
 559         c.Check(api.Calls, Equals, 2)
 560         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 561         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 562         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 563         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 564 }
 565
 566 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 567         api := &ArvTestClient{}
 568         kc := &KeepTestClient{}
 569         defer kc.Close()
 570         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 571         c.Assert(err, IsNil)
 572
 573         err = cr.UpdateContainerRunning()
 574         c.Check(err, IsNil)
 575
 576         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 577 }
 578
 579 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 580         api := &ArvTestClient{}
 581         kc := &KeepTestClient{}
 582         defer kc.Close()
 583         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 584         c.Assert(err, IsNil)
 585
 586         cr.LogsPDH = new(string)
 587         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 588
 589         cr.ExitCode = new(int)
 590         *cr.ExitCode = 42
 591         cr.finalState = "Complete"
 592
 593         err = cr.UpdateContainerFinal()
 594         c.Check(err, IsNil)
 595
 596         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 597         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 598         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 599 }
 600
 601 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 602         api := &ArvTestClient{}
 603         kc := &KeepTestClient{}
 604         defer kc.Close()
 605         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 606         c.Assert(err, IsNil)
 607         cr.cCancelled = true
 608         cr.finalState = "Cancelled"
 609
 610         err = cr.UpdateContainerFinal()
 611         c.Check(err, IsNil)
 612
 613         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 616 }
 617
 618 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 619 // dress rehearsal of the Run() function, starting from a JSON container record.
 620 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 621         err := json.Unmarshal([]byte(record), &s.api.Container)
 622         c.Assert(err, IsNil)
 623         initialState := s.api.Container.State
 624
 625         var sm struct {
 626                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 627         }
 628         err = json.Unmarshal([]byte(record), &sm)
 629         c.Check(err, IsNil)
 630         secretMounts, err := json.Marshal(sm)
 631         c.Assert(err, IsNil)
 632         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 633
 634         s.executor.runFunc = func() {
 635                 fn()
 636                 s.executor.exit <- exitCode
 637         }
 638
 639         s.runner.statInterval = 100 * time.Millisecond
 640         s.runner.containerWatchdogInterval = time.Second
 641         am := &ArvMountCmdLine{}
 642         s.runner.RunArvMount = am.ArvMountTest
 643
 644         realTemp := c.MkDir()
 645         tempcount := 0
 646         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 647                 tempcount++
 648                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 649                 err := os.Mkdir(d, os.ModePerm)
 650                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 651                         // Test case must have pre-populated the tempdir
 652                         err = nil
 653                 }
 654                 return d, err
 655         }
 656         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 657                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 658         }
 659
 660         if extraMounts != nil && len(extraMounts) > 0 {
 661                 err := s.runner.SetupArvMountPoint("keep")
 662                 c.Check(err, IsNil)
 663
 664                 for _, m := range extraMounts {
 665                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 666                 }
 667         }
 668
 669         err = s.runner.Run()
 670         if s.api.CalledWith("container.state", "Complete") != nil {
 671                 c.Check(err, IsNil)
 672         }
 673         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 674                 c.Check(s.api.WasSetRunning, Equals, true)
 675                 var lastupdate arvadosclient.Dict
 676                 for _, content := range s.api.Content {
 677                         if content["container"] != nil {
 678                                 lastupdate = content["container"].(arvadosclient.Dict)
 679                         }
 680                 }
 681                 if lastupdate["log"] == nil {
 682                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 683                 }
 684         }
 685
 686         if err != nil {
 687                 for k, v := range s.api.Logs {
 688                         c.Log(k)
 689                         c.Log(v.String())
 690                 }
 691         }
 692
 693         return s.api, s.runner, realTemp
 694 }
 695
 696 func (s *TestSuite) TestFullRunHello(c *C) {
 697         s.runner.enableMemoryLimit = true
 698         s.runner.networkMode = "default"
 699         s.fullRunHelper(c, `{
 700     "command": ["echo", "hello world"],
 701     "container_image": "`+arvadostest.DockerImage112PDH+`",
 702     "cwd": ".",
 703     "environment": {"foo":"bar","baz":"waz"},
 704     "mounts": {"/tmp": {"kind": "tmp"} },
 705     "output_path": "/tmp",
 706     "priority": 1,
 707     "runtime_constraints": {"vcpus":1,"ram":1000000},
 708     "state": "Locked",
 709     "output_storage_classes": ["default"]
 710 }`, nil, 0, func() {
 711                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 712                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 713                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 714                 c.Check(s.executor.created.VCPUs, Equals, 1)
 715                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 716                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 717                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 718                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 719                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 720         })
 721
 722         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 723         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 724         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 725         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 726         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 727 }
 728
 729 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 730         var ran bool
 731         s.fullRunHelper(c, `{
 732     "command": ["sleep", "3"],
 733     "container_image": "`+arvadostest.DockerImage112PDH+`",
 734     "cwd": ".",
 735     "environment": {},
 736     "mounts": {"/tmp": {"kind": "tmp"} },
 737     "output_path": "/tmp",
 738     "priority": 1,
 739     "runtime_constraints": {},
 740     "scheduling_parameters":{"max_run_time": 1},
 741     "state": "Running"
 742 }`, nil, 2, func() {
 743                 ran = true
 744         })
 745         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 746         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 747         c.Check(ran, Equals, false)
 748 }
 749
 750 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 751         s.fullRunHelper(c, `{
 752     "command": ["sleep", "3"],
 753     "container_image": "`+arvadostest.DockerImage112PDH+`",
 754     "cwd": ".",
 755     "environment": {},
 756     "mounts": {"/tmp": {"kind": "tmp"} },
 757     "output_path": "/tmp",
 758     "priority": 1,
 759     "runtime_constraints": {},
 760     "scheduling_parameters":{"max_run_time": 1},
 761     "state": "Locked"
 762 }`, nil, 0, func() {
 763                 time.Sleep(3 * time.Second)
 764         })
 765
 766         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 767         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 768 }
 769
 770 func (s *TestSuite) TestContainerWaitFails(c *C) {
 771         s.fullRunHelper(c, `{
 772     "command": ["sleep", "3"],
 773     "container_image": "`+arvadostest.DockerImage112PDH+`",
 774     "cwd": ".",
 775     "mounts": {"/tmp": {"kind": "tmp"} },
 776     "output_path": "/tmp",
 777     "priority": 1,
 778     "state": "Locked"
 779 }`, nil, 0, func() {
 780                 s.executor.waitErr = errors.New("Container is not running")
 781         })
 782
 783         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 784         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 785 }
 786
 787 func (s *TestSuite) TestCrunchstat(c *C) {
 788         s.fullRunHelper(c, `{
 789                 "command": ["sleep", "1"],
 790                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 791                 "cwd": ".",
 792                 "environment": {},
 793                 "mounts": {"/tmp": {"kind": "tmp"} },
 794                 "output_path": "/tmp",
 795                 "priority": 1,
 796                 "runtime_constraints": {},
 797                 "state": "Locked"
 798         }`, nil, 0, func() {
 799                 time.Sleep(time.Second)
 800         })
 801
 802         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 803         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 804
 805         // We didn't actually start a container, so crunchstat didn't
 806         // find accounting files and therefore didn't log any stats.
 807         // It should have logged a "can't find accounting files"
 808         // message after one poll interval, though, so we can confirm
 809         // it's alive:
 810         c.Assert(s.api.Logs["crunchstat"], NotNil)
 811         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 812
 813         // The "files never appeared" log assures us that we called
 814         // (*crunchstat.Reporter)Stop(), and that we set it up with
 815         // the correct container ID "abcde":
 816         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 817 }
 818
 819 func (s *TestSuite) TestNodeInfoLog(c *C) {
 820         os.Setenv("SLURMD_NODENAME", "compute2")
 821         s.fullRunHelper(c, `{
 822                 "command": ["sleep", "1"],
 823                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 824                 "cwd": ".",
 825                 "environment": {},
 826                 "mounts": {"/tmp": {"kind": "tmp"} },
 827                 "output_path": "/tmp",
 828                 "priority": 1,
 829                 "runtime_constraints": {},
 830                 "state": "Locked"
 831         }`, nil, 0,
 832                 func() {
 833                         time.Sleep(time.Second)
 834                 })
 835
 836         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 837         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 838
 839         c.Assert(s.api.Logs["node"], NotNil)
 840         json := s.api.Logs["node"].String()
 841         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 842         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 843         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 844
 845         c.Assert(s.api.Logs["node-info"], NotNil)
 846         json = s.api.Logs["node-info"].String()
 847         c.Check(json, Matches, `(?ms).*Host Information.*`)
 848         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 849         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 850         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 851         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 852 }
 853
 854 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 855         s.fullRunHelper(c, `{
 856                 "command": ["sleep", "1"],
 857                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 858                 "cwd": ".",
 859                 "environment": {},
 860                 "mounts": {"/tmp": {"kind": "tmp"} },
 861                 "output_path": "/tmp",
 862                 "priority": 1,
 863                 "runtime_constraints": {},
 864                 "state": "Locked"
 865         }`, nil, 0,
 866                 func() {
 867                 })
 868
 869         c.Assert(s.api.Logs["crunch-run"], NotNil)
 870         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 871         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 872 }
 873
 874 func (s *TestSuite) TestContainerRecordLog(c *C) {
 875         s.fullRunHelper(c, `{
 876                 "command": ["sleep", "1"],
 877                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 878                 "cwd": ".",
 879                 "environment": {},
 880                 "mounts": {"/tmp": {"kind": "tmp"} },
 881                 "output_path": "/tmp",
 882                 "priority": 1,
 883                 "runtime_constraints": {},
 884                 "state": "Locked"
 885         }`, nil, 0,
 886                 func() {
 887                         time.Sleep(time.Second)
 888                 })
 889
 890         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 891         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 892
 893         c.Assert(s.api.Logs["container"], NotNil)
 894         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 895 }
 896
 897 func (s *TestSuite) TestFullRunStderr(c *C) {
 898         s.fullRunHelper(c, `{
 899     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 900     "container_image": "`+arvadostest.DockerImage112PDH+`",
 901     "cwd": ".",
 902     "environment": {},
 903     "mounts": {"/tmp": {"kind": "tmp"} },
 904     "output_path": "/tmp",
 905     "priority": 1,
 906     "runtime_constraints": {},
 907     "state": "Locked"
 908 }`, nil, 1, func() {
 909                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 910                 fmt.Fprintln(s.executor.created.Stderr, "world")
 911         })
 912
 913         final := s.api.CalledWith("container.state", "Complete")
 914         c.Assert(final, NotNil)
 915         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 916         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 917
 918         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 919         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 920 }
 921
 922 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 923         s.fullRunHelper(c, `{
 924     "command": ["pwd"],
 925     "container_image": "`+arvadostest.DockerImage112PDH+`",
 926     "cwd": ".",
 927     "environment": {},
 928     "mounts": {"/tmp": {"kind": "tmp"} },
 929     "output_path": "/tmp",
 930     "priority": 1,
 931     "runtime_constraints": {},
 932     "state": "Locked"
 933 }`, nil, 0, func() {
 934                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 935         })
 936
 937         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 938         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 939         c.Log(s.api.Logs["stdout"])
 940         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 941 }
 942
 943 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 944         s.fullRunHelper(c, `{
 945     "command": ["pwd"],
 946     "container_image": "`+arvadostest.DockerImage112PDH+`",
 947     "cwd": "/bin",
 948     "environment": {},
 949     "mounts": {"/tmp": {"kind": "tmp"} },
 950     "output_path": "/tmp",
 951     "priority": 1,
 952     "runtime_constraints": {},
 953     "state": "Locked"
 954 }`, nil, 0, func() {
 955                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 956         })
 957
 958         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 959         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 960         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 961 }
 962
 963 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 964         s.fullRunHelper(c, `{
 965     "command": ["pwd"],
 966     "container_image": "`+arvadostest.DockerImage112PDH+`",
 967     "cwd": "/bin",
 968     "environment": {},
 969     "mounts": {"/tmp": {"kind": "tmp"} },
 970     "output_path": "/tmp",
 971     "priority": 1,
 972     "runtime_constraints": {},
 973     "state": "Locked",
 974     "output_storage_classes": ["foo", "bar"]
 975 }`, nil, 0, func() {
 976                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 977         })
 978
 979         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 980         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 981         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 982         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 983         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 984 }
 985
 986 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
 987         s.fullRunHelper(c, `{
 988     "command": ["pwd"],
 989     "container_image": "`+arvadostest.DockerImage112PDH+`",
 990     "cwd": "/bin",
 991     "environment": {},
 992     "mounts": {"/tmp": {"kind": "tmp"} },
 993     "output_path": "/tmp",
 994     "priority": 1,
 995     "runtime_constraints": {"cuda": {"device_count": 2}},
 996     "state": "Locked",
 997     "output_storage_classes": ["foo", "bar"]
 998 }`, nil, 0, func() {
 999                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1000         })
1001         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1002 }
1003
1004 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1005         s.fullRunHelper(c, `{
1006     "command": ["pwd"],
1007     "container_image": "`+arvadostest.DockerImage112PDH+`",
1008     "cwd": "/bin",
1009     "environment": {},
1010     "mounts": {"/tmp": {"kind": "tmp"} },
1011     "output_path": "/tmp",
1012     "priority": 1,
1013     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1014     "state": "Locked",
1015     "output_storage_classes": ["foo", "bar"]
1016 }`, nil, 0, func() {
1017                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1018         })
1019         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1020 }
1021
1022 func (s *TestSuite) TestStopOnSignal(c *C) {
1023         s.executor.runFunc = func() {
1024                 s.executor.created.Stdout.Write([]byte("foo\n"))
1025                 s.runner.SigChan <- syscall.SIGINT
1026         }
1027         s.testStopContainer(c)
1028 }
1029
1030 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1031         s.executor.runFunc = func() {
1032                 s.executor.created.Stdout.Write([]byte("foo\n"))
1033                 s.runner.ArvMountExit <- nil
1034                 close(s.runner.ArvMountExit)
1035         }
1036         s.runner.ArvMountExit = make(chan error)
1037         s.testStopContainer(c)
1038 }
1039
1040 func (s *TestSuite) testStopContainer(c *C) {
1041         record := `{
1042     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1043     "container_image": "` + arvadostest.DockerImage112PDH + `",
1044     "cwd": ".",
1045     "environment": {},
1046     "mounts": {"/tmp": {"kind": "tmp"} },
1047     "output_path": "/tmp",
1048     "priority": 1,
1049     "runtime_constraints": {},
1050     "state": "Locked"
1051 }`
1052
1053         err := json.Unmarshal([]byte(record), &s.api.Container)
1054         c.Assert(err, IsNil)
1055
1056         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1057         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1058                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1059         }
1060
1061         done := make(chan error)
1062         go func() {
1063                 done <- s.runner.Run()
1064         }()
1065         select {
1066         case <-time.After(20 * time.Second):
1067                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1068                 c.Fatal("timed out")
1069         case err = <-done:
1070                 c.Check(err, IsNil)
1071         }
1072         for k, v := range s.api.Logs {
1073                 c.Log(k)
1074                 c.Log(v.String(), "\n")
1075         }
1076
1077         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1078         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1079         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1080 }
1081
1082 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1083         s.fullRunHelper(c, `{
1084     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1085     "container_image": "`+arvadostest.DockerImage112PDH+`",
1086     "cwd": "/bin",
1087     "environment": {"FROBIZ": "bilbo"},
1088     "mounts": {"/tmp": {"kind": "tmp"} },
1089     "output_path": "/tmp",
1090     "priority": 1,
1091     "runtime_constraints": {},
1092     "state": "Locked"
1093 }`, nil, 0, func() {
1094                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1095         })
1096
1097         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1098         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1099         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1100 }
1101
1102 type ArvMountCmdLine struct {
1103         Cmd   []string
1104         token string
1105 }
1106
1107 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1108         am.Cmd = c
1109         am.token = token
1110         return nil, nil
1111 }
1112
1113 func stubCert(temp string) string {
1114         path := temp + "/ca-certificates.crt"
1115         crt, _ := os.Create(path)
1116         crt.Close()
1117         arvadosclient.CertFiles = []string{path}
1118         return path
1119 }
1120
1121 func (s *TestSuite) TestSetupMounts(c *C) {
1122         cr := s.runner
1123         am := &ArvMountCmdLine{}
1124         cr.RunArvMount = am.ArvMountTest
1125         cr.ContainerArvClient = &ArvTestClient{}
1126         cr.ContainerKeepClient = &KeepTestClient{}
1127         cr.Container.OutputStorageClasses = []string{"default"}
1128
1129         realTemp := c.MkDir()
1130         certTemp := c.MkDir()
1131         stubCertPath := stubCert(certTemp)
1132         cr.parentTemp = realTemp
1133
1134         i := 0
1135         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1136                 i++
1137                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1138                 err := os.Mkdir(d, os.ModePerm)
1139                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1140                         // Test case must have pre-populated the tempdir
1141                         err = nil
1142                 }
1143                 return d, err
1144         }
1145
1146         checkEmpty := func() {
1147                 // Should be deleted.
1148                 _, err := os.Stat(realTemp)
1149                 c.Assert(os.IsNotExist(err), Equals, true)
1150
1151                 // Now recreate it for the next test.
1152                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1153         }
1154
1155         {
1156                 i = 0
1157                 cr.ArvMountPoint = ""
1158                 cr.Container.Mounts = make(map[string]arvados.Mount)
1159                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1160                 cr.Container.OutputPath = "/tmp"
1161                 cr.statInterval = 5 * time.Second
1162                 bindmounts, err := cr.SetupMounts()
1163                 c.Check(err, IsNil)
1164                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1165                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1166                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1167                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1168                 os.RemoveAll(cr.ArvMountPoint)
1169                 cr.CleanupDirs()
1170                 checkEmpty()
1171         }
1172
1173         {
1174                 i = 0
1175                 cr.ArvMountPoint = ""
1176                 cr.Container.Mounts = make(map[string]arvados.Mount)
1177                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1178                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1179                 cr.Container.OutputPath = "/out"
1180                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1181
1182                 bindmounts, err := cr.SetupMounts()
1183                 c.Check(err, IsNil)
1184                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1185                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1186                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1187                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1188                 os.RemoveAll(cr.ArvMountPoint)
1189                 cr.CleanupDirs()
1190                 checkEmpty()
1191         }
1192
1193         {
1194                 i = 0
1195                 cr.ArvMountPoint = ""
1196                 cr.Container.Mounts = make(map[string]arvados.Mount)
1197                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1198                 cr.Container.OutputPath = "/tmp"
1199                 cr.Container.RuntimeConstraints.API = true
1200                 cr.Container.OutputStorageClasses = []string{"default"}
1201
1202                 bindmounts, err := cr.SetupMounts()
1203                 c.Check(err, IsNil)
1204                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1205                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1206                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1207                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1208                 os.RemoveAll(cr.ArvMountPoint)
1209                 cr.CleanupDirs()
1210                 checkEmpty()
1211
1212                 cr.Container.RuntimeConstraints.API = false
1213         }
1214
1215         {
1216                 i = 0
1217                 cr.ArvMountPoint = ""
1218                 cr.Container.Mounts = map[string]arvados.Mount{
1219                         "/keeptmp": {Kind: "collection", Writable: true},
1220                 }
1221                 cr.Container.OutputPath = "/keeptmp"
1222
1223                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1224
1225                 bindmounts, err := cr.SetupMounts()
1226                 c.Check(err, IsNil)
1227                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1228                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1229                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1230                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1231                 os.RemoveAll(cr.ArvMountPoint)
1232                 cr.CleanupDirs()
1233                 checkEmpty()
1234         }
1235
1236         {
1237                 i = 0
1238                 cr.ArvMountPoint = ""
1239                 cr.Container.Mounts = map[string]arvados.Mount{
1240                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1241                         "/keepout": {Kind: "collection", Writable: true},
1242                 }
1243                 cr.Container.OutputPath = "/keepout"
1244
1245                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1246                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1247
1248                 bindmounts, err := cr.SetupMounts()
1249                 c.Check(err, IsNil)
1250                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1251                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1252                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1253                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1254                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1255                         "/keepout": {realTemp + "/keep1/tmp0", false},
1256                 })
1257                 os.RemoveAll(cr.ArvMountPoint)
1258                 cr.CleanupDirs()
1259                 checkEmpty()
1260         }
1261
1262         {
1263                 i = 0
1264                 cr.ArvMountPoint = ""
1265                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1266                 cr.Container.Mounts = map[string]arvados.Mount{
1267                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1268                         "/keepout": {Kind: "collection", Writable: true},
1269                 }
1270                 cr.Container.OutputPath = "/keepout"
1271
1272                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1273                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1274
1275                 bindmounts, err := cr.SetupMounts()
1276                 c.Check(err, IsNil)
1277                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1278                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1279                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1280                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1281                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1282                         "/keepout": {realTemp + "/keep1/tmp0", false},
1283                 })
1284                 os.RemoveAll(cr.ArvMountPoint)
1285                 cr.CleanupDirs()
1286                 checkEmpty()
1287         }
1288
1289         for _, test := range []struct {
1290                 in  interface{}
1291                 out string
1292         }{
1293                 {in: "foo", out: `"foo"`},
1294                 {in: nil, out: `null`},
1295                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1296         } {
1297                 i = 0
1298                 cr.ArvMountPoint = ""
1299                 cr.Container.Mounts = map[string]arvados.Mount{
1300                         "/mnt/test.json": {Kind: "json", Content: test.in},
1301                 }
1302                 bindmounts, err := cr.SetupMounts()
1303                 c.Check(err, IsNil)
1304                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1305                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1306                 })
1307                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1308                 c.Check(err, IsNil)
1309                 c.Check(content, DeepEquals, []byte(test.out))
1310                 os.RemoveAll(cr.ArvMountPoint)
1311                 cr.CleanupDirs()
1312                 checkEmpty()
1313         }
1314
1315         for _, test := range []struct {
1316                 in  interface{}
1317                 out string
1318         }{
1319                 {in: "foo", out: `foo`},
1320                 {in: nil, out: "error"},
1321                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1322         } {
1323                 i = 0
1324                 cr.ArvMountPoint = ""
1325                 cr.Container.Mounts = map[string]arvados.Mount{
1326                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1327                 }
1328                 bindmounts, err := cr.SetupMounts()
1329                 if test.out == "error" {
1330                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1331                 } else {
1332                         c.Check(err, IsNil)
1333                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1334                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1335                         })
1336                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1337                         c.Check(err, IsNil)
1338                         c.Check(content, DeepEquals, []byte(test.out))
1339                 }
1340                 os.RemoveAll(cr.ArvMountPoint)
1341                 cr.CleanupDirs()
1342                 checkEmpty()
1343         }
1344
1345         // Read-only mount points are allowed underneath output_dir mount point
1346         {
1347                 i = 0
1348                 cr.ArvMountPoint = ""
1349                 cr.Container.Mounts = make(map[string]arvados.Mount)
1350                 cr.Container.Mounts = map[string]arvados.Mount{
1351                         "/tmp":     {Kind: "tmp"},
1352                         "/tmp/foo": {Kind: "collection"},
1353                 }
1354                 cr.Container.OutputPath = "/tmp"
1355
1356                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1357
1358                 bindmounts, err := cr.SetupMounts()
1359                 c.Check(err, IsNil)
1360                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1361                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1362                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1363                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1364                         "/tmp":     {realTemp + "/tmp2", false},
1365                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1366                 })
1367                 os.RemoveAll(cr.ArvMountPoint)
1368                 cr.CleanupDirs()
1369                 checkEmpty()
1370         }
1371
1372         // Writable mount points copied to output_dir mount point
1373         {
1374                 i = 0
1375                 cr.ArvMountPoint = ""
1376                 cr.Container.Mounts = make(map[string]arvados.Mount)
1377                 cr.Container.Mounts = map[string]arvados.Mount{
1378                         "/tmp": {Kind: "tmp"},
1379                         "/tmp/foo": {Kind: "collection",
1380                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1381                                 Writable:         true},
1382                         "/tmp/bar": {Kind: "collection",
1383                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1384                                 Path:             "baz",
1385                                 Writable:         true},
1386                 }
1387                 cr.Container.OutputPath = "/tmp"
1388
1389                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1390                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1391
1392                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1393                 rf.Write([]byte("bar"))
1394                 rf.Close()
1395
1396                 _, err := cr.SetupMounts()
1397                 c.Check(err, IsNil)
1398                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1399                 c.Check(err, IsNil)
1400                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1401                 c.Check(err, IsNil)
1402                 os.RemoveAll(cr.ArvMountPoint)
1403                 cr.CleanupDirs()
1404                 checkEmpty()
1405         }
1406
1407         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1408         {
1409                 i = 0
1410                 cr.ArvMountPoint = ""
1411                 cr.Container.Mounts = make(map[string]arvados.Mount)
1412                 cr.Container.Mounts = map[string]arvados.Mount{
1413                         "/tmp":     {Kind: "tmp"},
1414                         "/tmp/foo": {Kind: "tmp"},
1415                 }
1416                 cr.Container.OutputPath = "/tmp"
1417
1418                 _, err := cr.SetupMounts()
1419                 c.Check(err, NotNil)
1420                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1421                 os.RemoveAll(cr.ArvMountPoint)
1422                 cr.CleanupDirs()
1423                 checkEmpty()
1424         }
1425
1426         // Only mount point of kind 'collection' is allowed for stdin
1427         {
1428                 i = 0
1429                 cr.ArvMountPoint = ""
1430                 cr.Container.Mounts = make(map[string]arvados.Mount)
1431                 cr.Container.Mounts = map[string]arvados.Mount{
1432                         "stdin": {Kind: "tmp"},
1433                 }
1434
1435                 _, err := cr.SetupMounts()
1436                 c.Check(err, NotNil)
1437                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1438                 os.RemoveAll(cr.ArvMountPoint)
1439                 cr.CleanupDirs()
1440                 checkEmpty()
1441         }
1442
1443         // git_tree mounts
1444         {
1445                 i = 0
1446                 cr.ArvMountPoint = ""
1447                 (*GitMountSuite)(nil).useTestGitServer(c)
1448                 cr.token = arvadostest.ActiveToken
1449                 cr.Container.Mounts = make(map[string]arvados.Mount)
1450                 cr.Container.Mounts = map[string]arvados.Mount{
1451                         "/tip": {
1452                                 Kind:   "git_tree",
1453                                 UUID:   arvadostest.Repository2UUID,
1454                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1455                                 Path:   "/",
1456                         },
1457                         "/non-tip": {
1458                                 Kind:   "git_tree",
1459                                 UUID:   arvadostest.Repository2UUID,
1460                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1461                                 Path:   "/",
1462                         },
1463                 }
1464                 cr.Container.OutputPath = "/tmp"
1465
1466                 bindmounts, err := cr.SetupMounts()
1467                 c.Check(err, IsNil)
1468
1469                 for path, mount := range bindmounts {
1470                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1471                 }
1472
1473                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1474                 c.Check(err, IsNil)
1475                 c.Check(string(data), Equals, "\000\001\002\003")
1476                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1477                 c.Check(err, FitsTypeOf, &os.PathError{})
1478                 c.Check(os.IsNotExist(err), Equals, true)
1479
1480                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1481                 c.Check(err, IsNil)
1482                 c.Check(string(data), Equals, "\000\001\002\003")
1483                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1484                 c.Check(err, IsNil)
1485                 c.Check(string(data), Equals, "testfile\n")
1486
1487                 cr.CleanupDirs()
1488                 checkEmpty()
1489         }
1490 }
1491
1492 func (s *TestSuite) TestStdout(c *C) {
1493         helperRecord := `{
1494                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1495                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1496                 "cwd": "/bin",
1497                 "environment": {"FROBIZ": "bilbo"},
1498                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1499                 "output_path": "/tmp",
1500                 "priority": 1,
1501                 "runtime_constraints": {},
1502                 "state": "Locked"
1503         }`
1504
1505         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1506                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1507         })
1508
1509         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1510         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1511         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1512 }
1513
1514 // Used by the TestStdoutWithWrongPath*()
1515 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1516         err := json.Unmarshal([]byte(record), &s.api.Container)
1517         c.Assert(err, IsNil)
1518         s.executor.runFunc = fn
1519         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1520         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1521                 return s.api, &KeepTestClient{}, nil, nil
1522         }
1523         return s.api, s.runner, s.runner.Run()
1524 }
1525
1526 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1527         _, _, err := s.stdoutErrorRunHelper(c, `{
1528     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1529     "output_path": "/tmp",
1530     "state": "Locked"
1531 }`, func() {})
1532         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1533 }
1534
1535 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1536         _, _, err := s.stdoutErrorRunHelper(c, `{
1537     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1538     "output_path": "/tmp",
1539     "state": "Locked"
1540 }`, func() {})
1541         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1542 }
1543
1544 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1545         _, _, err := s.stdoutErrorRunHelper(c, `{
1546     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1547     "output_path": "/tmp",
1548     "state": "Locked"
1549 }`, func() {})
1550         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1551 }
1552
1553 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1554         s.fullRunHelper(c, `{
1555     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1556     "container_image": "`+arvadostest.DockerImage112PDH+`",
1557     "cwd": "/bin",
1558     "environment": {},
1559     "mounts": {"/tmp": {"kind": "tmp"} },
1560     "output_path": "/tmp",
1561     "priority": 1,
1562     "runtime_constraints": {"API": true},
1563     "state": "Locked"
1564 }`, nil, 0, func() {
1565                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1566                 s.executor.exit <- 3
1567         })
1568         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1569         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1570         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1571 }
1572
1573 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1574         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1575         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1576         s.fullRunHelper(c, `{
1577     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1578     "container_image": "`+arvadostest.DockerImage112PDH+`",
1579     "cwd": "/bin",
1580     "environment": {},
1581     "mounts": {"/tmp": {"kind": "tmp"} },
1582     "output_path": "/tmp",
1583     "priority": 1,
1584     "runtime_constraints": {"API": true},
1585     "state": "Locked"
1586 }`, nil, 0, func() {
1587                 s.api.Container.Output = arvadostest.DockerImage112PDH
1588         })
1589
1590         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1591         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1592         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1593 }
1594
1595 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1596         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1597                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1598                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1599                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1600         }
1601         s.executor.runFunc = func() {
1602                 time.Sleep(time.Second)
1603                 s.executor.exit <- 137
1604         }
1605         record := `{
1606     "command": ["sleep", "1"],
1607     "container_image": "` + arvadostest.DockerImage112PDH + `",
1608     "cwd": "/bin",
1609     "environment": {},
1610     "mounts": {"/tmp": {"kind": "tmp"} },
1611     "output_path": "/tmp",
1612     "priority": 1,
1613     "runtime_constraints": {"API": true},
1614     "state": "Locked"
1615 }`
1616         err := json.Unmarshal([]byte(record), &s.api.Container)
1617         c.Assert(err, IsNil)
1618         err = s.runner.Run()
1619         c.Assert(err, IsNil)
1620         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1621         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1622         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1623         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1624         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1625 }
1626
1627 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1628         helperRecord := `{
1629                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1630                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1631                 "cwd": "/bin",
1632                 "environment": {"FROBIZ": "bilbo"},
1633                 "mounts": {
1634         "/tmp": {"kind": "tmp"},
1635         "/tmp/foo": {"kind": "collection",
1636                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1637                      "exclude_from_output": true
1638         },
1639         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1640     },
1641                 "output_path": "/tmp",
1642                 "priority": 1,
1643                 "runtime_constraints": {},
1644                 "state": "Locked"
1645         }`
1646
1647         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1648
1649         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1650                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1651         })
1652
1653         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1654         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1655         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1656 }
1657
1658 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1659         helperRecord := `{
1660                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1661                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1662                 "cwd": "/bin",
1663                 "environment": {"FROBIZ": "bilbo"},
1664                 "mounts": {
1665         "/tmp": {"kind": "tmp"},
1666         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1667         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1668         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1669         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1670         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1671     },
1672                 "output_path": "/tmp",
1673                 "priority": 1,
1674                 "runtime_constraints": {},
1675                 "state": "Locked"
1676         }`
1677
1678         extraMounts := []string{
1679                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1680                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1681                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1682         }
1683
1684         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1685                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1686         })
1687
1688         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1689                 "/tmp":                   {realtemp + "/tmp1", false},
1690                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1691                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1692                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1693                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1694         })
1695
1696         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1697         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1698         for _, v := range api.Content {
1699                 if v["collection"] != nil {
1700                         c.Check(v["ensure_unique_name"], Equals, true)
1701                         collection := v["collection"].(arvadosclient.Dict)
1702                         if strings.Index(collection["name"].(string), "output") == 0 {
1703                                 manifest := collection["manifest_text"].(string)
1704
1705                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1706 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1707 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1708 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1709 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1710 `)
1711                         }
1712                 }
1713         }
1714 }
1715
1716 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1717         helperRecord := `{
1718                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1719                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1720                 "cwd": "/bin",
1721                 "environment": {"FROBIZ": "bilbo"},
1722                 "mounts": {
1723         "/tmp": {"kind": "tmp"},
1724         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1725         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1726     },
1727                 "output_path": "/tmp",
1728                 "priority": 1,
1729                 "runtime_constraints": {},
1730                 "state": "Locked"
1731         }`
1732
1733         extraMounts := []string{
1734                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1735         }
1736
1737         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1738                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1739         })
1740
1741         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1742         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1743         for _, v := range s.api.Content {
1744                 if v["collection"] != nil {
1745                         collection := v["collection"].(arvadosclient.Dict)
1746                         if strings.Index(collection["name"].(string), "output") == 0 {
1747                                 manifest := collection["manifest_text"].(string)
1748
1749                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1750 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1751 `)
1752                         }
1753                 }
1754         }
1755 }
1756
1757 func (s *TestSuite) TestOutputError(c *C) {
1758         helperRecord := `{
1759                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1760                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1761                 "cwd": "/bin",
1762                 "environment": {"FROBIZ": "bilbo"},
1763                 "mounts": {
1764                         "/tmp": {"kind": "tmp"}
1765                 },
1766                 "output_path": "/tmp",
1767                 "priority": 1,
1768                 "runtime_constraints": {},
1769                 "state": "Locked"
1770         }`
1771         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1772                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1773         })
1774
1775         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1776 }
1777
1778 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1779         helperRecord := `{
1780                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1781                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1782                 "cwd": "/bin",
1783                 "environment": {"FROBIZ": "bilbo"},
1784                 "mounts": {
1785         "/tmp": {"kind": "tmp"},
1786         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1787         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1788     },
1789                 "output_path": "/tmp",
1790                 "priority": 1,
1791                 "runtime_constraints": {},
1792                 "state": "Locked"
1793         }`
1794
1795         extraMounts := []string{
1796                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1797         }
1798
1799         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1800                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1801         })
1802
1803         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1804         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1805         for _, v := range api.Content {
1806                 if v["collection"] != nil {
1807                         collection := v["collection"].(arvadosclient.Dict)
1808                         if strings.Index(collection["name"].(string), "output") == 0 {
1809                                 manifest := collection["manifest_text"].(string)
1810                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1811 `)
1812                         }
1813                 }
1814         }
1815 }
1816
1817 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1818         helperRecord := `{
1819                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1820                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1821                 "cwd": "/bin",
1822                 "environment": {"FROBIZ": "bilbo"},
1823                 "mounts": {
1824         "/tmp": {"kind": "tmp"},
1825         "stdin": {"kind": "json", "content": "foo"},
1826         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1827     },
1828                 "output_path": "/tmp",
1829                 "priority": 1,
1830                 "runtime_constraints": {},
1831                 "state": "Locked"
1832         }`
1833
1834         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1835                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1836         })
1837
1838         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1839         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1840         for _, v := range api.Content {
1841                 if v["collection"] != nil {
1842                         collection := v["collection"].(arvadosclient.Dict)
1843                         if strings.Index(collection["name"].(string), "output") == 0 {
1844                                 manifest := collection["manifest_text"].(string)
1845                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1846 `)
1847                         }
1848                 }
1849         }
1850 }
1851
1852 func (s *TestSuite) TestStderrMount(c *C) {
1853         api, cr, _ := s.fullRunHelper(c, `{
1854     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1855     "container_image": "`+arvadostest.DockerImage112PDH+`",
1856     "cwd": ".",
1857     "environment": {},
1858     "mounts": {"/tmp": {"kind": "tmp"},
1859                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1860                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1861     "output_path": "/tmp",
1862     "priority": 1,
1863     "runtime_constraints": {},
1864     "state": "Locked"
1865 }`, nil, 1, func() {
1866                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1867                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1868         })
1869
1870         final := api.CalledWith("container.state", "Complete")
1871         c.Assert(final, NotNil)
1872         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1873         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1874
1875         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1876 }
1877
1878 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1879         s.api.callraw = true
1880         err := s.runner.fetchContainerRecord()
1881         c.Assert(err, IsNil)
1882         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1883         c.Logf("%#v", s.runner.Container)
1884         c.Check(err, IsNil)
1885         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1886 }
1887
1888 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1889         nextState := ""
1890         for _, setup := range []func(){
1891                 func() {
1892                         c.Log("// waitErr = ocl runtime error")
1893                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1894                         nextState = "Cancelled"
1895                 },
1896                 func() {
1897                         c.Log("// loadErr = cannot connect")
1898                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1899                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1900                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1901                         c.Assert(err, IsNil)
1902                         nextState = "Queued"
1903                 },
1904         } {
1905                 s.SetUpTest(c)
1906                 setup()
1907                 s.fullRunHelper(c, `{
1908     "command": ["echo", "hello world"],
1909     "container_image": "`+arvadostest.DockerImage112PDH+`",
1910     "cwd": ".",
1911     "environment": {},
1912     "mounts": {"/tmp": {"kind": "tmp"} },
1913     "output_path": "/tmp",
1914     "priority": 1,
1915     "runtime_constraints": {},
1916     "state": "Locked"
1917 }`, nil, 0, func() {})
1918                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1919                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1920                 if *brokenNodeHook != "" {
1921                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1922                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1923                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1924                 } else {
1925                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1926                 }
1927         }
1928 }
1929
1930 func (s *TestSuite) TestBadCommand(c *C) {
1931         for _, startError := range []string{
1932                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1933                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1934                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1935         } {
1936                 s.SetUpTest(c)
1937                 s.executor.startErr = errors.New(startError)
1938                 s.fullRunHelper(c, `{
1939     "command": ["echo", "hello world"],
1940     "container_image": "`+arvadostest.DockerImage112PDH+`",
1941     "cwd": ".",
1942     "environment": {},
1943     "mounts": {"/tmp": {"kind": "tmp"} },
1944     "output_path": "/tmp",
1945     "priority": 1,
1946     "runtime_constraints": {},
1947     "state": "Locked"
1948 }`, nil, 0, func() {})
1949                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1950                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1951         }
1952 }
1953
1954 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1955         helperRecord := `{
1956                 "command": ["true"],
1957                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1958                 "cwd": "/bin",
1959                 "mounts": {
1960                     "/tmp": {"kind": "tmp"},
1961                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1962                 },
1963                 "secret_mounts": {
1964                 },
1965                 "output_path": "/tmp",
1966                 "priority": 1,
1967                 "runtime_constraints": {},
1968                 "state": "Locked"
1969         }`
1970
1971         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1972                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1973                 c.Check(err, IsNil)
1974                 c.Check(string(content), Equals, "mypassword")
1975         })
1976
1977         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1978         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1979         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1980         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1981
1982         // under secret mounts, not captured in output
1983         helperRecord = `{
1984                 "command": ["true"],
1985                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1986                 "cwd": "/bin",
1987                 "mounts": {
1988                     "/tmp": {"kind": "tmp"}
1989                 },
1990                 "secret_mounts": {
1991                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1992                 },
1993                 "output_path": "/tmp",
1994                 "priority": 1,
1995                 "runtime_constraints": {},
1996                 "state": "Locked"
1997         }`
1998
1999         s.SetUpTest(c)
2000         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2001                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2002                 c.Check(err, IsNil)
2003                 c.Check(string(content), Equals, "mypassword")
2004         })
2005
2006         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2007         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2008         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2009         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2010 }
2011
2012 type FakeProcess struct {
2013         cmdLine []string
2014 }
2015
2016 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2017         return fp.cmdLine, nil
2018 }