lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         keepmountTmp             []string
  48         testDispatcherKeepClient KeepTestClient
  49         testContainerKeepClient  KeepTestClient
  50 }
  51
  52 func (s *TestSuite) SetUpTest(c *C) {
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 for i, opt := range cmd {
  66                         if opt == "--mount-tmp" {
  67                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  68                                 if err != nil {
  69                                         return nil, err
  70                                 }
  71                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  72                         }
  73                 }
  74                 return nil, nil
  75         }
  76         s.keepmount = c.MkDir()
  77         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  78         s.keepmountTmp = nil
  79         c.Assert(err, IsNil)
  80         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  81         c.Assert(err, IsNil)
  82         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  83         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  84         c.Assert(err, IsNil)
  85         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  86         c.Assert(err, IsNil)
  87         s.runner.ArvMountPoint = s.keepmount
  88 }
  89
  90 type ArvTestClient struct {
  91         Total   int64
  92         Calls   int
  93         Content []arvadosclient.Dict
  94         arvados.Container
  95         secretMounts []byte
  96         Logs         map[string]*bytes.Buffer
  97         sync.Mutex
  98         WasSetRunning bool
  99         callraw       bool
 100 }
 101
 102 type KeepTestClient struct {
 103         Called         bool
 104         Content        []byte
 105         StorageClasses []string
 106 }
 107
 108 type stubExecutor struct {
 109         imageLoaded bool
 110         loaded      string
 111         loadErr     error
 112         exitCode    int
 113         createErr   error
 114         created     containerSpec
 115         startErr    error
 116         waitSleep   time.Duration
 117         waitErr     error
 118         stopErr     error
 119         stopped     bool
 120         closed      bool
 121         runFunc     func()
 122         exit        chan int
 123 }
 124
 125 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 126         containerClient *arvados.Client) error {
 127         e.loaded = tarball
 128         return e.loadErr
 129 }
 130 func (e *stubExecutor) Runtime() string                 { return "stub" }
 131 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 132 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 133 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 134 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 135 func (e *stubExecutor) Close()                          { e.closed = true }
 136 func (e *stubExecutor) Wait(context.Context) (int, error) {
 137         return <-e.exit, e.waitErr
 138 }
 139
 140 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 141
 142 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 143 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 144 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 145
 146 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 147 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 148
 149 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 150 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 151 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 152 `
 153
 154 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 155
 156 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 157 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 158
 159 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 160 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 161
 162 func (client *ArvTestClient) Create(resourceType string,
 163         parameters arvadosclient.Dict,
 164         output interface{}) error {
 165
 166         client.Mutex.Lock()
 167         defer client.Mutex.Unlock()
 168
 169         client.Calls++
 170         client.Content = append(client.Content, parameters)
 171
 172         if resourceType == "logs" {
 173                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 174                 if client.Logs == nil {
 175                         client.Logs = make(map[string]*bytes.Buffer)
 176                 }
 177                 if client.Logs[et] == nil {
 178                         client.Logs[et] = &bytes.Buffer{}
 179                 }
 180                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 181         }
 182
 183         if resourceType == "collections" && output != nil {
 184                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 185                 outmap := output.(*arvados.Collection)
 186                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 187                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 188         }
 189
 190         return nil
 191 }
 192
 193 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 194         switch {
 195         case method == "GET" && resourceType == "containers" && action == "auth":
 196                 return json.Unmarshal([]byte(`{
 197                         "kind": "arvados#api_client_authorization",
 198                         "uuid": "`+fakeAuthUUID+`",
 199                         "api_token": "`+fakeAuthToken+`"
 200                         }`), output)
 201         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 202                 if client.secretMounts != nil {
 203                         return json.Unmarshal(client.secretMounts, output)
 204                 }
 205                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 206         default:
 207                 return fmt.Errorf("Not found")
 208         }
 209 }
 210
 211 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 212         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 213         var j []byte
 214         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 215                 j = []byte(`{
 216                         "kind": "arvados#nodeList",
 217                         "items": [{
 218                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 219                                 "hostname": "compute2",
 220                                 "properties": {"total_cpu_cores": 16}
 221                         }]}`)
 222         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 223                 if uuid == "" {
 224                         j, err = json.Marshal(map[string]interface{}{
 225                                 "items": []interface{}{client.Container},
 226                                 "kind":  "arvados#nodeList",
 227                         })
 228                 } else {
 229                         j, err = json.Marshal(client.Container)
 230                 }
 231         } else {
 232                 j = []byte(`{
 233                         "command": ["sleep", "1"],
 234                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 235                         "cwd": ".",
 236                         "environment": {},
 237                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 238                         "output_path": "/tmp",
 239                         "priority": 1,
 240                         "runtime_constraints": {}
 241                 }`)
 242         }
 243         return ioutil.NopCloser(bytes.NewReader(j)), err
 244 }
 245
 246 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 247         if resourceType == "collections" {
 248                 if uuid == hwPDH {
 249                         output.(*arvados.Collection).ManifestText = hwManifest
 250                 } else if uuid == otherPDH {
 251                         output.(*arvados.Collection).ManifestText = otherManifest
 252                 } else if uuid == normalizedWithSubdirsPDH {
 253                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 254                 } else if uuid == denormalizedWithSubdirsPDH {
 255                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 256                 }
 257         }
 258         if resourceType == "containers" {
 259                 (*output.(*arvados.Container)) = client.Container
 260         }
 261         return nil
 262 }
 263
 264 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 265         client.Mutex.Lock()
 266         defer client.Mutex.Unlock()
 267         client.Calls++
 268         client.Content = append(client.Content, parameters)
 269         if resourceType == "containers" {
 270                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 271                         client.WasSetRunning = true
 272                 }
 273         } else if resourceType == "collections" {
 274                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 275                 output.(*arvados.Collection).UUID = uuid
 276                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 277         }
 278         return nil
 279 }
 280
 281 var discoveryMap = map[string]interface{}{
 282         "defaultTrashLifetime":               float64(1209600),
 283         "crunchLimitLogBytesPerJob":          float64(67108864),
 284         "crunchLogThrottleBytes":             float64(65536),
 285         "crunchLogThrottlePeriod":            float64(60),
 286         "crunchLogThrottleLines":             float64(1024),
 287         "crunchLogPartialLineThrottlePeriod": float64(5),
 288         "crunchLogBytesPerEvent":             float64(4096),
 289         "crunchLogSecondsBetweenEvents":      float64(1),
 290 }
 291
 292 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 293         return discoveryMap[key], nil
 294 }
 295
 296 // CalledWith returns the parameters from the first API call whose
 297 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 298 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 299 // no call matches, it returns nil.
 300 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 301 call:
 302         for _, content := range client.Content {
 303                 var v interface{} = content
 304                 for _, k := range strings.Split(jpath, ".") {
 305                         if dict, ok := v.(arvadosclient.Dict); !ok {
 306                                 continue call
 307                         } else {
 308                                 v = dict[k]
 309                         }
 310                 }
 311                 if v == expect {
 312                         return content
 313                 }
 314         }
 315         return nil
 316 }
 317
 318 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 319         return locator, nil
 320 }
 321
 322 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 323         client.Content = opts.Data
 324         return arvados.BlockWriteResponse{
 325                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 326         }, nil
 327 }
 328
 329 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 330         return 0, errors.New("not implemented")
 331 }
 332
 333 func (client *KeepTestClient) ClearBlockCache() {
 334 }
 335
 336 func (client *KeepTestClient) Close() {
 337         client.Content = nil
 338 }
 339
 340 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 341         client.StorageClasses = sc
 342 }
 343
 344 type FileWrapper struct {
 345         io.ReadCloser
 346         len int64
 347 }
 348
 349 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 350         return nil, errors.New("not implemented")
 351 }
 352
 353 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 354         return 0, errors.New("not implemented")
 355 }
 356
 357 func (fw FileWrapper) Size() int64 {
 358         return fw.len
 359 }
 360
 361 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 362         return nil, errors.New("not implemented")
 363 }
 364
 365 func (fw FileWrapper) Truncate(int64) error {
 366         return errors.New("not implemented")
 367 }
 368
 369 func (fw FileWrapper) Write([]byte) (int, error) {
 370         return 0, errors.New("not implemented")
 371 }
 372
 373 func (fw FileWrapper) Sync() error {
 374         return errors.New("not implemented")
 375 }
 376
 377 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 378         return nil, errors.New("not implemented")
 379 }
 380
 381 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 382         return errors.New("not implemented")
 383 }
 384
 385 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 386         if filename == hwImageID+".tar" {
 387                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 388                 client.Called = true
 389                 return FileWrapper{rdr, 1321984}, nil
 390         } else if filename == "/file1_in_main.txt" {
 391                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 392                 client.Called = true
 393                 return FileWrapper{rdr, 3}, nil
 394         }
 395         return nil, nil
 396 }
 397
 398 func (s *TestSuite) TestLoadImage(c *C) {
 399         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 400         s.runner.Container.Mounts = map[string]arvados.Mount{
 401                 "/out": {Kind: "tmp", Writable: true},
 402         }
 403         s.runner.Container.OutputPath = "/out"
 404
 405         _, err := s.runner.SetupMounts()
 406         c.Assert(err, IsNil)
 407
 408         imageID, err := s.runner.LoadImage()
 409         c.Check(err, IsNil)
 410         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 411         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 412
 413         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 414         s.executor.imageLoaded = false
 415         s.executor.loaded = ""
 416         s.executor.loadErr = errors.New("bork")
 417         imageID, err = s.runner.LoadImage()
 418         c.Check(err, ErrorMatches, ".*bork")
 419         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 420
 421         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 422         s.executor.imageLoaded = false
 423         s.executor.loaded = ""
 424         s.executor.loadErr = nil
 425         imageID, err = s.runner.LoadImage()
 426         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 427         c.Check(s.executor.loaded, Equals, "")
 428 }
 429
 430 type ArvErrorTestClient struct{}
 431
 432 func (ArvErrorTestClient) Create(resourceType string,
 433         parameters arvadosclient.Dict,
 434         output interface{}) error {
 435         return nil
 436 }
 437
 438 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 439         if method == "GET" && resourceType == "containers" && action == "auth" {
 440                 return nil
 441         }
 442         return errors.New("ArvError")
 443 }
 444
 445 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 446         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 447         return nil, errors.New("ArvError")
 448 }
 449
 450 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 451         return errors.New("ArvError")
 452 }
 453
 454 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 455         return nil
 456 }
 457
 458 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 459         return discoveryMap[key], nil
 460 }
 461
 462 type KeepErrorTestClient struct {
 463         KeepTestClient
 464 }
 465
 466 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 467         return nil, errors.New("KeepError")
 468 }
 469
 470 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 471         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 472 }
 473
 474 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 475         return "", errors.New("KeepError")
 476 }
 477
 478 type KeepReadErrorTestClient struct {
 479         KeepTestClient
 480 }
 481
 482 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 483         return 0, errors.New("KeepError")
 484 }
 485
 486 type ErrorReader struct {
 487         FileWrapper
 488 }
 489
 490 func (ErrorReader) Read(p []byte) (n int, err error) {
 491         return 0, errors.New("ErrorReader")
 492 }
 493
 494 func (ErrorReader) Seek(int64, int) (int64, error) {
 495         return 0, errors.New("ErrorReader")
 496 }
 497
 498 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 499         return ErrorReader{}, nil
 500 }
 501
 502 type ClosableBuffer struct {
 503         bytes.Buffer
 504 }
 505
 506 func (*ClosableBuffer) Close() error {
 507         return nil
 508 }
 509
 510 type TestLogs struct {
 511         Stdout ClosableBuffer
 512         Stderr ClosableBuffer
 513 }
 514
 515 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 516         if logstr == "stdout" {
 517                 return &tl.Stdout, nil
 518         }
 519         if logstr == "stderr" {
 520                 return &tl.Stderr, nil
 521         }
 522         return nil, errors.New("???")
 523 }
 524
 525 func dockerLog(fd byte, msg string) []byte {
 526         by := []byte(msg)
 527         header := make([]byte, 8+len(by))
 528         header[0] = fd
 529         header[7] = byte(len(by))
 530         copy(header[8:], by)
 531         return header
 532 }
 533
 534 func (s *TestSuite) TestRunContainer(c *C) {
 535         s.executor.runFunc = func() {
 536                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 537                 s.executor.exit <- 0
 538         }
 539
 540         var logs TestLogs
 541         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 542         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 543         s.runner.Container.Command = []string{"./hw"}
 544         s.runner.Container.OutputStorageClasses = []string{"default"}
 545
 546         imageID, err := s.runner.LoadImage()
 547         c.Assert(err, IsNil)
 548
 549         err = s.runner.CreateContainer(imageID, nil)
 550         c.Assert(err, IsNil)
 551
 552         err = s.runner.StartContainer()
 553         c.Assert(err, IsNil)
 554
 555         err = s.runner.WaitFinish()
 556         c.Assert(err, IsNil)
 557
 558         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 559         c.Check(logs.Stderr.String(), Equals, "")
 560 }
 561
 562 func (s *TestSuite) TestCommitLogs(c *C) {
 563         api := &ArvTestClient{}
 564         kc := &KeepTestClient{}
 565         defer kc.Close()
 566         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 567         c.Assert(err, IsNil)
 568         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 569
 570         cr.CrunchLog.Print("Hello world!")
 571         cr.CrunchLog.Print("Goodbye")
 572         cr.finalState = "Complete"
 573
 574         err = cr.CommitLogs()
 575         c.Check(err, IsNil)
 576
 577         c.Check(api.Calls, Equals, 2)
 578         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 579         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 580         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 581         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 582 }
 583
 584 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 585         api := &ArvTestClient{}
 586         kc := &KeepTestClient{}
 587         defer kc.Close()
 588         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 589         c.Assert(err, IsNil)
 590
 591         err = cr.UpdateContainerRunning()
 592         c.Check(err, IsNil)
 593
 594         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 595 }
 596
 597 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 598         api := &ArvTestClient{}
 599         kc := &KeepTestClient{}
 600         defer kc.Close()
 601         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 602         c.Assert(err, IsNil)
 603
 604         cr.LogsPDH = new(string)
 605         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 606
 607         cr.ExitCode = new(int)
 608         *cr.ExitCode = 42
 609         cr.finalState = "Complete"
 610
 611         err = cr.UpdateContainerFinal()
 612         c.Check(err, IsNil)
 613
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 616         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 617 }
 618
 619 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 620         api := &ArvTestClient{}
 621         kc := &KeepTestClient{}
 622         defer kc.Close()
 623         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 624         c.Assert(err, IsNil)
 625         cr.cCancelled = true
 626         cr.finalState = "Cancelled"
 627
 628         err = cr.UpdateContainerFinal()
 629         c.Check(err, IsNil)
 630
 631         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 632         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 633         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 634 }
 635
 636 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 637 // dress rehearsal of the Run() function, starting from a JSON container record.
 638 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 639         err := json.Unmarshal([]byte(record), &s.api.Container)
 640         c.Assert(err, IsNil)
 641         initialState := s.api.Container.State
 642
 643         var sm struct {
 644                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 645         }
 646         err = json.Unmarshal([]byte(record), &sm)
 647         c.Check(err, IsNil)
 648         secretMounts, err := json.Marshal(sm)
 649         c.Assert(err, IsNil)
 650         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 651
 652         s.executor.runFunc = func() {
 653                 fn()
 654                 s.executor.exit <- exitCode
 655         }
 656
 657         s.runner.statInterval = 100 * time.Millisecond
 658         s.runner.containerWatchdogInterval = time.Second
 659
 660         realTemp := c.MkDir()
 661         tempcount := 0
 662         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 663                 tempcount++
 664                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 665                 err := os.Mkdir(d, os.ModePerm)
 666                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 667                         // Test case must have pre-populated the tempdir
 668                         err = nil
 669                 }
 670                 return d, err
 671         }
 672         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 673                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 674         }
 675
 676         if extraMounts != nil && len(extraMounts) > 0 {
 677                 err := s.runner.SetupArvMountPoint("keep")
 678                 c.Check(err, IsNil)
 679
 680                 for _, m := range extraMounts {
 681                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 682                 }
 683         }
 684
 685         err = s.runner.Run()
 686         if s.api.CalledWith("container.state", "Complete") != nil {
 687                 c.Check(err, IsNil)
 688         }
 689         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 690                 c.Check(s.api.WasSetRunning, Equals, true)
 691                 var lastupdate arvadosclient.Dict
 692                 for _, content := range s.api.Content {
 693                         if content["container"] != nil {
 694                                 lastupdate = content["container"].(arvadosclient.Dict)
 695                         }
 696                 }
 697                 if lastupdate["log"] == nil {
 698                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 699                 }
 700         }
 701
 702         if err != nil {
 703                 for k, v := range s.api.Logs {
 704                         c.Log(k)
 705                         c.Log(v.String())
 706                 }
 707         }
 708
 709         return s.api, s.runner, realTemp
 710 }
 711
 712 func (s *TestSuite) TestFullRunHello(c *C) {
 713         s.runner.enableMemoryLimit = true
 714         s.runner.networkMode = "default"
 715         s.fullRunHelper(c, `{
 716     "command": ["echo", "hello world"],
 717     "container_image": "`+arvadostest.DockerImage112PDH+`",
 718     "cwd": ".",
 719     "environment": {"foo":"bar","baz":"waz"},
 720     "mounts": {"/tmp": {"kind": "tmp"} },
 721     "output_path": "/tmp",
 722     "priority": 1,
 723     "runtime_constraints": {"vcpus":1,"ram":1000000},
 724     "state": "Locked",
 725     "output_storage_classes": ["default"]
 726 }`, nil, 0, func() {
 727                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 728                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 729                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 730                 c.Check(s.executor.created.VCPUs, Equals, 1)
 731                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 732                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 733                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 734                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 735                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 736         })
 737
 738         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 739         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 740         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 741         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 742         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 743 }
 744
 745 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 746         var ran bool
 747         s.fullRunHelper(c, `{
 748     "command": ["sleep", "3"],
 749     "container_image": "`+arvadostest.DockerImage112PDH+`",
 750     "cwd": ".",
 751     "environment": {},
 752     "mounts": {"/tmp": {"kind": "tmp"} },
 753     "output_path": "/tmp",
 754     "priority": 1,
 755     "runtime_constraints": {},
 756     "scheduling_parameters":{"max_run_time": 1},
 757     "state": "Running"
 758 }`, nil, 2, func() {
 759                 ran = true
 760         })
 761         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 762         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 763         c.Check(ran, Equals, false)
 764 }
 765
 766 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 767         s.fullRunHelper(c, `{
 768     "command": ["sleep", "3"],
 769     "container_image": "`+arvadostest.DockerImage112PDH+`",
 770     "cwd": ".",
 771     "environment": {},
 772     "mounts": {"/tmp": {"kind": "tmp"} },
 773     "output_path": "/tmp",
 774     "priority": 1,
 775     "runtime_constraints": {},
 776     "scheduling_parameters":{"max_run_time": 1},
 777     "state": "Locked"
 778 }`, nil, 0, func() {
 779                 time.Sleep(3 * time.Second)
 780         })
 781
 782         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 783         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 784 }
 785
 786 func (s *TestSuite) TestContainerWaitFails(c *C) {
 787         s.fullRunHelper(c, `{
 788     "command": ["sleep", "3"],
 789     "container_image": "`+arvadostest.DockerImage112PDH+`",
 790     "cwd": ".",
 791     "mounts": {"/tmp": {"kind": "tmp"} },
 792     "output_path": "/tmp",
 793     "priority": 1,
 794     "state": "Locked"
 795 }`, nil, 0, func() {
 796                 s.executor.waitErr = errors.New("Container is not running")
 797         })
 798
 799         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 800         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 801 }
 802
 803 func (s *TestSuite) TestCrunchstat(c *C) {
 804         s.fullRunHelper(c, `{
 805                 "command": ["sleep", "1"],
 806                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 807                 "cwd": ".",
 808                 "environment": {},
 809                 "mounts": {"/tmp": {"kind": "tmp"} },
 810                 "output_path": "/tmp",
 811                 "priority": 1,
 812                 "runtime_constraints": {},
 813                 "state": "Locked"
 814         }`, nil, 0, func() {
 815                 time.Sleep(time.Second)
 816         })
 817
 818         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 819         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 820
 821         // We didn't actually start a container, so crunchstat didn't
 822         // find accounting files and therefore didn't log any stats.
 823         // It should have logged a "can't find accounting files"
 824         // message after one poll interval, though, so we can confirm
 825         // it's alive:
 826         c.Assert(s.api.Logs["crunchstat"], NotNil)
 827         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 828
 829         // The "files never appeared" log assures us that we called
 830         // (*crunchstat.Reporter)Stop(), and that we set it up with
 831         // the correct container ID "abcde":
 832         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 833 }
 834
 835 func (s *TestSuite) TestNodeInfoLog(c *C) {
 836         os.Setenv("SLURMD_NODENAME", "compute2")
 837         s.fullRunHelper(c, `{
 838                 "command": ["sleep", "1"],
 839                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 840                 "cwd": ".",
 841                 "environment": {},
 842                 "mounts": {"/tmp": {"kind": "tmp"} },
 843                 "output_path": "/tmp",
 844                 "priority": 1,
 845                 "runtime_constraints": {},
 846                 "state": "Locked"
 847         }`, nil, 0,
 848                 func() {
 849                         time.Sleep(time.Second)
 850                 })
 851
 852         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 853         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 854
 855         c.Assert(s.api.Logs["node"], NotNil)
 856         json := s.api.Logs["node"].String()
 857         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 858         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 859         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 860
 861         c.Assert(s.api.Logs["node-info"], NotNil)
 862         json = s.api.Logs["node-info"].String()
 863         c.Check(json, Matches, `(?ms).*Host Information.*`)
 864         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 865         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 866         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 867         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 868 }
 869
 870 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 871         s.fullRunHelper(c, `{
 872                 "command": ["sleep", "1"],
 873                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 874                 "cwd": ".",
 875                 "environment": {},
 876                 "mounts": {"/tmp": {"kind": "tmp"} },
 877                 "output_path": "/tmp",
 878                 "priority": 1,
 879                 "runtime_constraints": {},
 880                 "state": "Locked"
 881         }`, nil, 0,
 882                 func() {
 883                 })
 884
 885         c.Assert(s.api.Logs["crunch-run"], NotNil)
 886         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 887         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 888         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 889 }
 890
 891 func (s *TestSuite) TestContainerRecordLog(c *C) {
 892         s.fullRunHelper(c, `{
 893                 "command": ["sleep", "1"],
 894                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 895                 "cwd": ".",
 896                 "environment": {},
 897                 "mounts": {"/tmp": {"kind": "tmp"} },
 898                 "output_path": "/tmp",
 899                 "priority": 1,
 900                 "runtime_constraints": {},
 901                 "state": "Locked"
 902         }`, nil, 0,
 903                 func() {
 904                         time.Sleep(time.Second)
 905                 })
 906
 907         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 908         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 909
 910         c.Assert(s.api.Logs["container"], NotNil)
 911         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 912 }
 913
 914 func (s *TestSuite) TestFullRunStderr(c *C) {
 915         s.fullRunHelper(c, `{
 916     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 917     "container_image": "`+arvadostest.DockerImage112PDH+`",
 918     "cwd": ".",
 919     "environment": {},
 920     "mounts": {"/tmp": {"kind": "tmp"} },
 921     "output_path": "/tmp",
 922     "priority": 1,
 923     "runtime_constraints": {},
 924     "state": "Locked"
 925 }`, nil, 1, func() {
 926                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 927                 fmt.Fprintln(s.executor.created.Stderr, "world")
 928         })
 929
 930         final := s.api.CalledWith("container.state", "Complete")
 931         c.Assert(final, NotNil)
 932         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 933         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 934
 935         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 936         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 937 }
 938
 939 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 940         s.fullRunHelper(c, `{
 941     "command": ["pwd"],
 942     "container_image": "`+arvadostest.DockerImage112PDH+`",
 943     "cwd": ".",
 944     "environment": {},
 945     "mounts": {"/tmp": {"kind": "tmp"} },
 946     "output_path": "/tmp",
 947     "priority": 1,
 948     "runtime_constraints": {},
 949     "state": "Locked"
 950 }`, nil, 0, func() {
 951                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 952         })
 953
 954         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 955         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 956         c.Log(s.api.Logs["stdout"])
 957         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 958 }
 959
 960 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 961         s.fullRunHelper(c, `{
 962     "command": ["pwd"],
 963     "container_image": "`+arvadostest.DockerImage112PDH+`",
 964     "cwd": "/bin",
 965     "environment": {},
 966     "mounts": {"/tmp": {"kind": "tmp"} },
 967     "output_path": "/tmp",
 968     "priority": 1,
 969     "runtime_constraints": {},
 970     "state": "Locked"
 971 }`, nil, 0, func() {
 972                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 973         })
 974
 975         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 976         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 977         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 978 }
 979
 980 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 981         s.fullRunHelper(c, `{
 982     "command": ["pwd"],
 983     "container_image": "`+arvadostest.DockerImage112PDH+`",
 984     "cwd": "/bin",
 985     "environment": {},
 986     "mounts": {"/tmp": {"kind": "tmp"} },
 987     "output_path": "/tmp",
 988     "priority": 1,
 989     "runtime_constraints": {},
 990     "state": "Locked",
 991     "output_storage_classes": ["foo", "bar"]
 992 }`, nil, 0, func() {
 993                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 994         })
 995
 996         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 997         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 998         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 999         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1000         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1001 }
1002
1003 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1004         s.fullRunHelper(c, `{
1005     "command": ["pwd"],
1006     "container_image": "`+arvadostest.DockerImage112PDH+`",
1007     "cwd": "/bin",
1008     "environment": {},
1009     "mounts": {"/tmp": {"kind": "tmp"} },
1010     "output_path": "/tmp",
1011     "priority": 1,
1012     "runtime_constraints": {"cuda": {"device_count": 2}},
1013     "state": "Locked",
1014     "output_storage_classes": ["foo", "bar"]
1015 }`, nil, 0, func() {
1016                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1017         })
1018         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1019 }
1020
1021 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1022         s.fullRunHelper(c, `{
1023     "command": ["pwd"],
1024     "container_image": "`+arvadostest.DockerImage112PDH+`",
1025     "cwd": "/bin",
1026     "environment": {},
1027     "mounts": {"/tmp": {"kind": "tmp"} },
1028     "output_path": "/tmp",
1029     "priority": 1,
1030     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1031     "state": "Locked",
1032     "output_storage_classes": ["foo", "bar"]
1033 }`, nil, 0, func() {
1034                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1035         })
1036         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1037 }
1038
1039 func (s *TestSuite) TestStopOnSignal(c *C) {
1040         s.executor.runFunc = func() {
1041                 s.executor.created.Stdout.Write([]byte("foo\n"))
1042                 s.runner.SigChan <- syscall.SIGINT
1043         }
1044         s.testStopContainer(c)
1045 }
1046
1047 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1048         s.executor.runFunc = func() {
1049                 s.executor.created.Stdout.Write([]byte("foo\n"))
1050                 s.runner.ArvMountExit <- nil
1051                 close(s.runner.ArvMountExit)
1052         }
1053         s.runner.ArvMountExit = make(chan error)
1054         s.testStopContainer(c)
1055 }
1056
1057 func (s *TestSuite) testStopContainer(c *C) {
1058         record := `{
1059     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1060     "container_image": "` + arvadostest.DockerImage112PDH + `",
1061     "cwd": ".",
1062     "environment": {},
1063     "mounts": {"/tmp": {"kind": "tmp"} },
1064     "output_path": "/tmp",
1065     "priority": 1,
1066     "runtime_constraints": {},
1067     "state": "Locked"
1068 }`
1069
1070         err := json.Unmarshal([]byte(record), &s.api.Container)
1071         c.Assert(err, IsNil)
1072
1073         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1074         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1075                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1076         }
1077
1078         done := make(chan error)
1079         go func() {
1080                 done <- s.runner.Run()
1081         }()
1082         select {
1083         case <-time.After(20 * time.Second):
1084                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1085                 c.Fatal("timed out")
1086         case err = <-done:
1087                 c.Check(err, IsNil)
1088         }
1089         for k, v := range s.api.Logs {
1090                 c.Log(k)
1091                 c.Log(v.String(), "\n")
1092         }
1093
1094         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1095         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1096         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1097 }
1098
1099 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1100         s.fullRunHelper(c, `{
1101     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1102     "container_image": "`+arvadostest.DockerImage112PDH+`",
1103     "cwd": "/bin",
1104     "environment": {"FROBIZ": "bilbo"},
1105     "mounts": {"/tmp": {"kind": "tmp"} },
1106     "output_path": "/tmp",
1107     "priority": 1,
1108     "runtime_constraints": {},
1109     "state": "Locked"
1110 }`, nil, 0, func() {
1111                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1112         })
1113
1114         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1115         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1116         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1117 }
1118
1119 type ArvMountCmdLine struct {
1120         Cmd   []string
1121         token string
1122 }
1123
1124 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1125         am.Cmd = c
1126         am.token = token
1127         return nil, nil
1128 }
1129
1130 func stubCert(temp string) string {
1131         path := temp + "/ca-certificates.crt"
1132         crt, _ := os.Create(path)
1133         crt.Close()
1134         arvadosclient.CertFiles = []string{path}
1135         return path
1136 }
1137
1138 func (s *TestSuite) TestSetupMounts(c *C) {
1139         cr := s.runner
1140         am := &ArvMountCmdLine{}
1141         cr.RunArvMount = am.ArvMountTest
1142         cr.ContainerArvClient = &ArvTestClient{}
1143         cr.ContainerKeepClient = &KeepTestClient{}
1144         cr.Container.OutputStorageClasses = []string{"default"}
1145
1146         realTemp := c.MkDir()
1147         certTemp := c.MkDir()
1148         stubCertPath := stubCert(certTemp)
1149         cr.parentTemp = realTemp
1150
1151         i := 0
1152         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1153                 i++
1154                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1155                 err := os.Mkdir(d, os.ModePerm)
1156                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1157                         // Test case must have pre-populated the tempdir
1158                         err = nil
1159                 }
1160                 return d, err
1161         }
1162
1163         checkEmpty := func() {
1164                 // Should be deleted.
1165                 _, err := os.Stat(realTemp)
1166                 c.Assert(os.IsNotExist(err), Equals, true)
1167
1168                 // Now recreate it for the next test.
1169                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1170         }
1171
1172         {
1173                 i = 0
1174                 cr.ArvMountPoint = ""
1175                 cr.Container.Mounts = make(map[string]arvados.Mount)
1176                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1177                 cr.Container.OutputPath = "/tmp"
1178                 cr.statInterval = 5 * time.Second
1179                 bindmounts, err := cr.SetupMounts()
1180                 c.Check(err, IsNil)
1181                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1182                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1183                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1184                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1185                 os.RemoveAll(cr.ArvMountPoint)
1186                 cr.CleanupDirs()
1187                 checkEmpty()
1188         }
1189
1190         {
1191                 i = 0
1192                 cr.ArvMountPoint = ""
1193                 cr.Container.Mounts = make(map[string]arvados.Mount)
1194                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1195                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1196                 cr.Container.OutputPath = "/out"
1197                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1198
1199                 bindmounts, err := cr.SetupMounts()
1200                 c.Check(err, IsNil)
1201                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1202                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1203                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1204                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1205                 os.RemoveAll(cr.ArvMountPoint)
1206                 cr.CleanupDirs()
1207                 checkEmpty()
1208         }
1209
1210         {
1211                 i = 0
1212                 cr.ArvMountPoint = ""
1213                 cr.Container.Mounts = make(map[string]arvados.Mount)
1214                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1215                 cr.Container.OutputPath = "/tmp"
1216                 cr.Container.RuntimeConstraints.API = true
1217                 cr.Container.OutputStorageClasses = []string{"default"}
1218
1219                 bindmounts, err := cr.SetupMounts()
1220                 c.Check(err, IsNil)
1221                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1222                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1223                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1224                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1225                 os.RemoveAll(cr.ArvMountPoint)
1226                 cr.CleanupDirs()
1227                 checkEmpty()
1228
1229                 cr.Container.RuntimeConstraints.API = false
1230         }
1231
1232         {
1233                 i = 0
1234                 cr.ArvMountPoint = ""
1235                 cr.Container.Mounts = map[string]arvados.Mount{
1236                         "/keeptmp": {Kind: "collection", Writable: true},
1237                 }
1238                 cr.Container.OutputPath = "/keeptmp"
1239
1240                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1241
1242                 bindmounts, err := cr.SetupMounts()
1243                 c.Check(err, IsNil)
1244                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1245                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1246                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1247                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1248                 os.RemoveAll(cr.ArvMountPoint)
1249                 cr.CleanupDirs()
1250                 checkEmpty()
1251         }
1252
1253         {
1254                 i = 0
1255                 cr.ArvMountPoint = ""
1256                 cr.Container.Mounts = map[string]arvados.Mount{
1257                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1258                         "/keepout": {Kind: "collection", Writable: true},
1259                 }
1260                 cr.Container.OutputPath = "/keepout"
1261
1262                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1263                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1264
1265                 bindmounts, err := cr.SetupMounts()
1266                 c.Check(err, IsNil)
1267                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1268                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1269                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1270                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1271                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1272                         "/keepout": {realTemp + "/keep1/tmp0", false},
1273                 })
1274                 os.RemoveAll(cr.ArvMountPoint)
1275                 cr.CleanupDirs()
1276                 checkEmpty()
1277         }
1278
1279         {
1280                 i = 0
1281                 cr.ArvMountPoint = ""
1282                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1283                 cr.Container.Mounts = map[string]arvados.Mount{
1284                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1285                         "/keepout": {Kind: "collection", Writable: true},
1286                 }
1287                 cr.Container.OutputPath = "/keepout"
1288
1289                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1290                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1291
1292                 bindmounts, err := cr.SetupMounts()
1293                 c.Check(err, IsNil)
1294                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1295                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1296                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1297                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1298                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1299                         "/keepout": {realTemp + "/keep1/tmp0", false},
1300                 })
1301                 os.RemoveAll(cr.ArvMountPoint)
1302                 cr.CleanupDirs()
1303                 checkEmpty()
1304         }
1305
1306         for _, test := range []struct {
1307                 in  interface{}
1308                 out string
1309         }{
1310                 {in: "foo", out: `"foo"`},
1311                 {in: nil, out: `null`},
1312                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1313         } {
1314                 i = 0
1315                 cr.ArvMountPoint = ""
1316                 cr.Container.Mounts = map[string]arvados.Mount{
1317                         "/mnt/test.json": {Kind: "json", Content: test.in},
1318                 }
1319                 bindmounts, err := cr.SetupMounts()
1320                 c.Check(err, IsNil)
1321                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1322                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1323                 })
1324                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1325                 c.Check(err, IsNil)
1326                 c.Check(content, DeepEquals, []byte(test.out))
1327                 os.RemoveAll(cr.ArvMountPoint)
1328                 cr.CleanupDirs()
1329                 checkEmpty()
1330         }
1331
1332         for _, test := range []struct {
1333                 in  interface{}
1334                 out string
1335         }{
1336                 {in: "foo", out: `foo`},
1337                 {in: nil, out: "error"},
1338                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1339         } {
1340                 i = 0
1341                 cr.ArvMountPoint = ""
1342                 cr.Container.Mounts = map[string]arvados.Mount{
1343                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1344                 }
1345                 bindmounts, err := cr.SetupMounts()
1346                 if test.out == "error" {
1347                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1348                 } else {
1349                         c.Check(err, IsNil)
1350                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1351                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1352                         })
1353                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1354                         c.Check(err, IsNil)
1355                         c.Check(content, DeepEquals, []byte(test.out))
1356                 }
1357                 os.RemoveAll(cr.ArvMountPoint)
1358                 cr.CleanupDirs()
1359                 checkEmpty()
1360         }
1361
1362         // Read-only mount points are allowed underneath output_dir mount point
1363         {
1364                 i = 0
1365                 cr.ArvMountPoint = ""
1366                 cr.Container.Mounts = make(map[string]arvados.Mount)
1367                 cr.Container.Mounts = map[string]arvados.Mount{
1368                         "/tmp":     {Kind: "tmp"},
1369                         "/tmp/foo": {Kind: "collection"},
1370                 }
1371                 cr.Container.OutputPath = "/tmp"
1372
1373                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1374
1375                 bindmounts, err := cr.SetupMounts()
1376                 c.Check(err, IsNil)
1377                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1378                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1379                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1380                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1381                         "/tmp":     {realTemp + "/tmp2", false},
1382                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1383                 })
1384                 os.RemoveAll(cr.ArvMountPoint)
1385                 cr.CleanupDirs()
1386                 checkEmpty()
1387         }
1388
1389         // Writable mount points copied to output_dir mount point
1390         {
1391                 i = 0
1392                 cr.ArvMountPoint = ""
1393                 cr.Container.Mounts = make(map[string]arvados.Mount)
1394                 cr.Container.Mounts = map[string]arvados.Mount{
1395                         "/tmp": {Kind: "tmp"},
1396                         "/tmp/foo": {Kind: "collection",
1397                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1398                                 Writable:         true},
1399                         "/tmp/bar": {Kind: "collection",
1400                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1401                                 Path:             "baz",
1402                                 Writable:         true},
1403                 }
1404                 cr.Container.OutputPath = "/tmp"
1405
1406                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1407                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1408
1409                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1410                 rf.Write([]byte("bar"))
1411                 rf.Close()
1412
1413                 _, err := cr.SetupMounts()
1414                 c.Check(err, IsNil)
1415                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1416                 c.Check(err, IsNil)
1417                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1418                 c.Check(err, IsNil)
1419                 os.RemoveAll(cr.ArvMountPoint)
1420                 cr.CleanupDirs()
1421                 checkEmpty()
1422         }
1423
1424         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1425         {
1426                 i = 0
1427                 cr.ArvMountPoint = ""
1428                 cr.Container.Mounts = make(map[string]arvados.Mount)
1429                 cr.Container.Mounts = map[string]arvados.Mount{
1430                         "/tmp":     {Kind: "tmp"},
1431                         "/tmp/foo": {Kind: "tmp"},
1432                 }
1433                 cr.Container.OutputPath = "/tmp"
1434
1435                 _, err := cr.SetupMounts()
1436                 c.Check(err, NotNil)
1437                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1438                 os.RemoveAll(cr.ArvMountPoint)
1439                 cr.CleanupDirs()
1440                 checkEmpty()
1441         }
1442
1443         // Only mount point of kind 'collection' is allowed for stdin
1444         {
1445                 i = 0
1446                 cr.ArvMountPoint = ""
1447                 cr.Container.Mounts = make(map[string]arvados.Mount)
1448                 cr.Container.Mounts = map[string]arvados.Mount{
1449                         "stdin": {Kind: "tmp"},
1450                 }
1451
1452                 _, err := cr.SetupMounts()
1453                 c.Check(err, NotNil)
1454                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1455                 os.RemoveAll(cr.ArvMountPoint)
1456                 cr.CleanupDirs()
1457                 checkEmpty()
1458         }
1459
1460         // git_tree mounts
1461         {
1462                 i = 0
1463                 cr.ArvMountPoint = ""
1464                 (*GitMountSuite)(nil).useTestGitServer(c)
1465                 cr.token = arvadostest.ActiveToken
1466                 cr.Container.Mounts = make(map[string]arvados.Mount)
1467                 cr.Container.Mounts = map[string]arvados.Mount{
1468                         "/tip": {
1469                                 Kind:   "git_tree",
1470                                 UUID:   arvadostest.Repository2UUID,
1471                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1472                                 Path:   "/",
1473                         },
1474                         "/non-tip": {
1475                                 Kind:   "git_tree",
1476                                 UUID:   arvadostest.Repository2UUID,
1477                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1478                                 Path:   "/",
1479                         },
1480                 }
1481                 cr.Container.OutputPath = "/tmp"
1482
1483                 bindmounts, err := cr.SetupMounts()
1484                 c.Check(err, IsNil)
1485
1486                 for path, mount := range bindmounts {
1487                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1488                 }
1489
1490                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1491                 c.Check(err, IsNil)
1492                 c.Check(string(data), Equals, "\000\001\002\003")
1493                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1494                 c.Check(err, FitsTypeOf, &os.PathError{})
1495                 c.Check(os.IsNotExist(err), Equals, true)
1496
1497                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1498                 c.Check(err, IsNil)
1499                 c.Check(string(data), Equals, "\000\001\002\003")
1500                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1501                 c.Check(err, IsNil)
1502                 c.Check(string(data), Equals, "testfile\n")
1503
1504                 cr.CleanupDirs()
1505                 checkEmpty()
1506         }
1507 }
1508
1509 func (s *TestSuite) TestStdout(c *C) {
1510         helperRecord := `{
1511                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1512                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1513                 "cwd": "/bin",
1514                 "environment": {"FROBIZ": "bilbo"},
1515                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1516                 "output_path": "/tmp",
1517                 "priority": 1,
1518                 "runtime_constraints": {},
1519                 "state": "Locked"
1520         }`
1521
1522         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1523                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1524         })
1525
1526         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1527         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1528         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1529 }
1530
1531 // Used by the TestStdoutWithWrongPath*()
1532 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1533         err := json.Unmarshal([]byte(record), &s.api.Container)
1534         c.Assert(err, IsNil)
1535         s.executor.runFunc = fn
1536         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1537         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1538                 return s.api, &KeepTestClient{}, nil, nil
1539         }
1540         return s.api, s.runner, s.runner.Run()
1541 }
1542
1543 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1544         _, _, err := s.stdoutErrorRunHelper(c, `{
1545     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1546     "output_path": "/tmp",
1547     "state": "Locked"
1548 }`, func() {})
1549         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1550 }
1551
1552 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1553         _, _, err := s.stdoutErrorRunHelper(c, `{
1554     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1555     "output_path": "/tmp",
1556     "state": "Locked"
1557 }`, func() {})
1558         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1559 }
1560
1561 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1562         _, _, err := s.stdoutErrorRunHelper(c, `{
1563     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1564     "output_path": "/tmp",
1565     "state": "Locked"
1566 }`, func() {})
1567         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1568 }
1569
1570 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1571         s.fullRunHelper(c, `{
1572     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1573     "container_image": "`+arvadostest.DockerImage112PDH+`",
1574     "cwd": "/bin",
1575     "environment": {},
1576     "mounts": {"/tmp": {"kind": "tmp"} },
1577     "output_path": "/tmp",
1578     "priority": 1,
1579     "runtime_constraints": {"API": true},
1580     "state": "Locked"
1581 }`, nil, 0, func() {
1582                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1583                 s.executor.exit <- 3
1584         })
1585         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1586         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1587         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1588 }
1589
1590 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1591         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1592         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1593         s.fullRunHelper(c, `{
1594     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1595     "container_image": "`+arvadostest.DockerImage112PDH+`",
1596     "cwd": "/bin",
1597     "environment": {},
1598     "mounts": {"/tmp": {"kind": "tmp"} },
1599     "output_path": "/tmp",
1600     "priority": 1,
1601     "runtime_constraints": {"API": true},
1602     "state": "Locked"
1603 }`, nil, 0, func() {
1604                 s.api.Container.Output = arvadostest.DockerImage112PDH
1605         })
1606
1607         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1608         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1609         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1610 }
1611
1612 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1613         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1614                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1615                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1616                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1617         }
1618         s.executor.runFunc = func() {
1619                 time.Sleep(time.Second)
1620                 s.executor.exit <- 137
1621         }
1622         record := `{
1623     "command": ["sleep", "1"],
1624     "container_image": "` + arvadostest.DockerImage112PDH + `",
1625     "cwd": "/bin",
1626     "environment": {},
1627     "mounts": {"/tmp": {"kind": "tmp"} },
1628     "output_path": "/tmp",
1629     "priority": 1,
1630     "runtime_constraints": {"API": true},
1631     "state": "Locked"
1632 }`
1633         err := json.Unmarshal([]byte(record), &s.api.Container)
1634         c.Assert(err, IsNil)
1635         err = s.runner.Run()
1636         c.Assert(err, IsNil)
1637         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1638         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1639         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1640         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1641         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1642 }
1643
1644 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1645         helperRecord := `{
1646                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1647                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1648                 "cwd": "/bin",
1649                 "environment": {"FROBIZ": "bilbo"},
1650                 "mounts": {
1651         "/tmp": {"kind": "tmp"},
1652         "/tmp/foo": {"kind": "collection",
1653                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1654                      "exclude_from_output": true
1655         },
1656         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1657     },
1658                 "output_path": "/tmp",
1659                 "priority": 1,
1660                 "runtime_constraints": {},
1661                 "state": "Locked"
1662         }`
1663
1664         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1665
1666         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1667                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1668         })
1669
1670         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1671         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1672         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1673 }
1674
1675 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1676         helperRecord := `{
1677                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1678                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1679                 "cwd": "/bin",
1680                 "environment": {"FROBIZ": "bilbo"},
1681                 "mounts": {
1682         "/tmp": {"kind": "tmp"},
1683         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1684         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1685         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1686         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1687         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1688     },
1689                 "output_path": "/tmp",
1690                 "priority": 1,
1691                 "runtime_constraints": {},
1692                 "state": "Locked"
1693         }`
1694
1695         extraMounts := []string{
1696                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1697                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1698                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1699         }
1700
1701         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1702                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1703         })
1704
1705         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1706                 "/tmp":                   {realtemp + "/tmp1", false},
1707                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1708                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1709                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1710                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1711         })
1712
1713         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1714         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1715         for _, v := range api.Content {
1716                 if v["collection"] != nil {
1717                         c.Check(v["ensure_unique_name"], Equals, true)
1718                         collection := v["collection"].(arvadosclient.Dict)
1719                         if strings.Index(collection["name"].(string), "output") == 0 {
1720                                 manifest := collection["manifest_text"].(string)
1721
1722                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1723 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1724 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1725 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1726 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1727 `)
1728                         }
1729                 }
1730         }
1731 }
1732
1733 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1734         helperRecord := `{
1735                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1736                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1737                 "cwd": "/bin",
1738                 "environment": {"FROBIZ": "bilbo"},
1739                 "mounts": {
1740         "/tmp": {"kind": "tmp"},
1741         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1742         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1743     },
1744                 "output_path": "/tmp",
1745                 "priority": 1,
1746                 "runtime_constraints": {},
1747                 "state": "Locked"
1748         }`
1749
1750         extraMounts := []string{
1751                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1752         }
1753
1754         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1755                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1756         })
1757
1758         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1759         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1760         for _, v := range s.api.Content {
1761                 if v["collection"] != nil {
1762                         collection := v["collection"].(arvadosclient.Dict)
1763                         if strings.Index(collection["name"].(string), "output") == 0 {
1764                                 manifest := collection["manifest_text"].(string)
1765
1766                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1767 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1768 `)
1769                         }
1770                 }
1771         }
1772 }
1773
1774 func (s *TestSuite) TestOutputError(c *C) {
1775         helperRecord := `{
1776                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1777                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1778                 "cwd": "/bin",
1779                 "environment": {"FROBIZ": "bilbo"},
1780                 "mounts": {
1781                         "/tmp": {"kind": "tmp"}
1782                 },
1783                 "output_path": "/tmp",
1784                 "priority": 1,
1785                 "runtime_constraints": {},
1786                 "state": "Locked"
1787         }`
1788         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1789                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1790         })
1791
1792         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1793 }
1794
1795 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1796         helperRecord := `{
1797                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1798                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1799                 "cwd": "/bin",
1800                 "environment": {"FROBIZ": "bilbo"},
1801                 "mounts": {
1802         "/tmp": {"kind": "tmp"},
1803         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1804         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1805     },
1806                 "output_path": "/tmp",
1807                 "priority": 1,
1808                 "runtime_constraints": {},
1809                 "state": "Locked"
1810         }`
1811
1812         extraMounts := []string{
1813                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1814         }
1815
1816         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1817                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1818         })
1819
1820         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1821         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1822         for _, v := range api.Content {
1823                 if v["collection"] != nil {
1824                         collection := v["collection"].(arvadosclient.Dict)
1825                         if strings.Index(collection["name"].(string), "output") == 0 {
1826                                 manifest := collection["manifest_text"].(string)
1827                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1828 `)
1829                         }
1830                 }
1831         }
1832 }
1833
1834 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1835         helperRecord := `{
1836                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1837                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1838                 "cwd": "/bin",
1839                 "environment": {"FROBIZ": "bilbo"},
1840                 "mounts": {
1841         "/tmp": {"kind": "tmp"},
1842         "stdin": {"kind": "json", "content": "foo"},
1843         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1844     },
1845                 "output_path": "/tmp",
1846                 "priority": 1,
1847                 "runtime_constraints": {},
1848                 "state": "Locked"
1849         }`
1850
1851         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1852                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1853         })
1854
1855         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1856         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1857         for _, v := range api.Content {
1858                 if v["collection"] != nil {
1859                         collection := v["collection"].(arvadosclient.Dict)
1860                         if strings.Index(collection["name"].(string), "output") == 0 {
1861                                 manifest := collection["manifest_text"].(string)
1862                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1863 `)
1864                         }
1865                 }
1866         }
1867 }
1868
1869 func (s *TestSuite) TestStderrMount(c *C) {
1870         api, cr, _ := s.fullRunHelper(c, `{
1871     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1872     "container_image": "`+arvadostest.DockerImage112PDH+`",
1873     "cwd": ".",
1874     "environment": {},
1875     "mounts": {"/tmp": {"kind": "tmp"},
1876                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1877                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1878     "output_path": "/tmp",
1879     "priority": 1,
1880     "runtime_constraints": {},
1881     "state": "Locked"
1882 }`, nil, 1, func() {
1883                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1884                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1885         })
1886
1887         final := api.CalledWith("container.state", "Complete")
1888         c.Assert(final, NotNil)
1889         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1890         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1891
1892         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1893 }
1894
1895 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1896         s.api.callraw = true
1897         err := s.runner.fetchContainerRecord()
1898         c.Assert(err, IsNil)
1899         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1900         c.Logf("%#v", s.runner.Container)
1901         c.Check(err, IsNil)
1902         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1903 }
1904
1905 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1906         nextState := ""
1907         for _, setup := range []func(){
1908                 func() {
1909                         c.Log("// waitErr = ocl runtime error")
1910                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1911                         nextState = "Cancelled"
1912                 },
1913                 func() {
1914                         c.Log("// loadErr = cannot connect")
1915                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1916                         s.runner.brokenNodeHook = c.MkDir() + "/broken-node-hook"
1917                         err := ioutil.WriteFile(s.runner.brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1918                         c.Assert(err, IsNil)
1919                         nextState = "Queued"
1920                 },
1921         } {
1922                 s.SetUpTest(c)
1923                 setup()
1924                 s.fullRunHelper(c, `{
1925     "command": ["echo", "hello world"],
1926     "container_image": "`+arvadostest.DockerImage112PDH+`",
1927     "cwd": ".",
1928     "environment": {},
1929     "mounts": {"/tmp": {"kind": "tmp"} },
1930     "output_path": "/tmp",
1931     "priority": 1,
1932     "runtime_constraints": {},
1933     "state": "Locked"
1934 }`, nil, 0, func() {})
1935                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1936                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1937                 if s.runner.brokenNodeHook != "" {
1938                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1939                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1940                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1941                 } else {
1942                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1943                 }
1944         }
1945 }
1946
1947 func (s *TestSuite) TestBadCommand(c *C) {
1948         for _, startError := range []string{
1949                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1950                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1951                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1952         } {
1953                 s.SetUpTest(c)
1954                 s.executor.startErr = errors.New(startError)
1955                 s.fullRunHelper(c, `{
1956     "command": ["echo", "hello world"],
1957     "container_image": "`+arvadostest.DockerImage112PDH+`",
1958     "cwd": ".",
1959     "environment": {},
1960     "mounts": {"/tmp": {"kind": "tmp"} },
1961     "output_path": "/tmp",
1962     "priority": 1,
1963     "runtime_constraints": {},
1964     "state": "Locked"
1965 }`, nil, 0, func() {})
1966                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1967                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1968         }
1969 }
1970
1971 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1972         helperRecord := `{
1973                 "command": ["true"],
1974                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1975                 "cwd": "/bin",
1976                 "mounts": {
1977                     "/tmp": {"kind": "tmp"},
1978                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1979                 },
1980                 "secret_mounts": {
1981                 },
1982                 "output_path": "/tmp",
1983                 "priority": 1,
1984                 "runtime_constraints": {},
1985                 "state": "Locked"
1986         }`
1987
1988         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1989                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1990                 c.Check(err, IsNil)
1991                 c.Check(string(content), Equals, "mypassword")
1992         })
1993
1994         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1995         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1996         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1997         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1998
1999         // under secret mounts, not captured in output
2000         helperRecord = `{
2001                 "command": ["true"],
2002                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2003                 "cwd": "/bin",
2004                 "mounts": {
2005                     "/tmp": {"kind": "tmp"}
2006                 },
2007                 "secret_mounts": {
2008                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2009                 },
2010                 "output_path": "/tmp",
2011                 "priority": 1,
2012                 "runtime_constraints": {},
2013                 "state": "Locked"
2014         }`
2015
2016         s.SetUpTest(c)
2017         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2018                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2019                 c.Check(err, IsNil)
2020                 c.Check(string(content), Equals, "mypassword")
2021         })
2022
2023         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2024         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2025         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2026         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2027
2028         // under secret mounts, output dir is a collection, not captured in output
2029         helperRecord = `{
2030                 "command": ["true"],
2031                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2032                 "cwd": "/bin",
2033                 "mounts": {
2034                     "/tmp": {"kind": "collection", "writable": true}
2035                 },
2036                 "secret_mounts": {
2037                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2038                 },
2039                 "output_path": "/tmp",
2040                 "priority": 1,
2041                 "runtime_constraints": {},
2042                 "state": "Locked"
2043         }`
2044
2045         s.SetUpTest(c)
2046         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2047                 // secret.conf should be provisioned as a separate
2048                 // bind mount, i.e., it should not appear in the
2049                 // (fake) fuse filesystem as viewed from the host.
2050                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2051                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2052                         c.Logf("secret.conf: content %q, err %#v", content, err)
2053                 }
2054                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2055                 c.Check(err, IsNil)
2056         })
2057
2058         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2059         c.Check(err, IsNil)
2060         c.Check(string(content), Equals, "mypassword")
2061         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2062         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2063         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2064         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2065 }
2066
2067 type FakeProcess struct {
2068         cmdLine []string
2069 }
2070
2071 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2072         return fp.cmdLine, nil
2073 }