lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         keepmountTmp             []string
  48         testDispatcherKeepClient KeepTestClient
  49         testContainerKeepClient  KeepTestClient
  50 }
  51
  52 func (s *TestSuite) SetUpTest(c *C) {
  53         *brokenNodeHook = ""
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 133 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 134 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 135 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 136 func (e *stubExecutor) Close()                          { e.closed = true }
 137 func (e *stubExecutor) Wait(context.Context) (int, error) {
 138         return <-e.exit, e.waitErr
 139 }
 140
 141 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 142
 143 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 144 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 145 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 146
 147 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 148 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 149
 150 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 151 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 152 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 153 `
 154
 155 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 156
 157 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 158 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 159
 160 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 161 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 162
 163 func (client *ArvTestClient) Create(resourceType string,
 164         parameters arvadosclient.Dict,
 165         output interface{}) error {
 166
 167         client.Mutex.Lock()
 168         defer client.Mutex.Unlock()
 169
 170         client.Calls++
 171         client.Content = append(client.Content, parameters)
 172
 173         if resourceType == "logs" {
 174                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 175                 if client.Logs == nil {
 176                         client.Logs = make(map[string]*bytes.Buffer)
 177                 }
 178                 if client.Logs[et] == nil {
 179                         client.Logs[et] = &bytes.Buffer{}
 180                 }
 181                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 182         }
 183
 184         if resourceType == "collections" && output != nil {
 185                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 186                 outmap := output.(*arvados.Collection)
 187                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 188                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 189         }
 190
 191         return nil
 192 }
 193
 194 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 195         switch {
 196         case method == "GET" && resourceType == "containers" && action == "auth":
 197                 return json.Unmarshal([]byte(`{
 198                         "kind": "arvados#api_client_authorization",
 199                         "uuid": "`+fakeAuthUUID+`",
 200                         "api_token": "`+fakeAuthToken+`"
 201                         }`), output)
 202         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 203                 if client.secretMounts != nil {
 204                         return json.Unmarshal(client.secretMounts, output)
 205                 }
 206                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 207         default:
 208                 return fmt.Errorf("Not found")
 209         }
 210 }
 211
 212 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 213         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 214         var j []byte
 215         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 216                 j = []byte(`{
 217                         "kind": "arvados#nodeList",
 218                         "items": [{
 219                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 220                                 "hostname": "compute2",
 221                                 "properties": {"total_cpu_cores": 16}
 222                         }]}`)
 223         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 224                 if uuid == "" {
 225                         j, err = json.Marshal(map[string]interface{}{
 226                                 "items": []interface{}{client.Container},
 227                                 "kind":  "arvados#nodeList",
 228                         })
 229                 } else {
 230                         j, err = json.Marshal(client.Container)
 231                 }
 232         } else {
 233                 j = []byte(`{
 234                         "command": ["sleep", "1"],
 235                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 236                         "cwd": ".",
 237                         "environment": {},
 238                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 239                         "output_path": "/tmp",
 240                         "priority": 1,
 241                         "runtime_constraints": {}
 242                 }`)
 243         }
 244         return ioutil.NopCloser(bytes.NewReader(j)), err
 245 }
 246
 247 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 248         if resourceType == "collections" {
 249                 if uuid == hwPDH {
 250                         output.(*arvados.Collection).ManifestText = hwManifest
 251                 } else if uuid == otherPDH {
 252                         output.(*arvados.Collection).ManifestText = otherManifest
 253                 } else if uuid == normalizedWithSubdirsPDH {
 254                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 255                 } else if uuid == denormalizedWithSubdirsPDH {
 256                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 257                 }
 258         }
 259         if resourceType == "containers" {
 260                 (*output.(*arvados.Container)) = client.Container
 261         }
 262         return nil
 263 }
 264
 265 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 266         client.Mutex.Lock()
 267         defer client.Mutex.Unlock()
 268         client.Calls++
 269         client.Content = append(client.Content, parameters)
 270         if resourceType == "containers" {
 271                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 272                         client.WasSetRunning = true
 273                 }
 274         } else if resourceType == "collections" {
 275                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 276                 output.(*arvados.Collection).UUID = uuid
 277                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 278         }
 279         return nil
 280 }
 281
 282 var discoveryMap = map[string]interface{}{
 283         "defaultTrashLifetime":               float64(1209600),
 284         "crunchLimitLogBytesPerJob":          float64(67108864),
 285         "crunchLogThrottleBytes":             float64(65536),
 286         "crunchLogThrottlePeriod":            float64(60),
 287         "crunchLogThrottleLines":             float64(1024),
 288         "crunchLogPartialLineThrottlePeriod": float64(5),
 289         "crunchLogBytesPerEvent":             float64(4096),
 290         "crunchLogSecondsBetweenEvents":      float64(1),
 291 }
 292
 293 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 294         return discoveryMap[key], nil
 295 }
 296
 297 // CalledWith returns the parameters from the first API call whose
 298 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 299 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 300 // no call matches, it returns nil.
 301 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 302 call:
 303         for _, content := range client.Content {
 304                 var v interface{} = content
 305                 for _, k := range strings.Split(jpath, ".") {
 306                         if dict, ok := v.(arvadosclient.Dict); !ok {
 307                                 continue call
 308                         } else {
 309                                 v = dict[k]
 310                         }
 311                 }
 312                 if v == expect {
 313                         return content
 314                 }
 315         }
 316         return nil
 317 }
 318
 319 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 320         return locator, nil
 321 }
 322
 323 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 324         client.Content = opts.Data
 325         return arvados.BlockWriteResponse{
 326                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 327         }, nil
 328 }
 329
 330 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 331         return 0, errors.New("not implemented")
 332 }
 333
 334 func (client *KeepTestClient) ClearBlockCache() {
 335 }
 336
 337 func (client *KeepTestClient) Close() {
 338         client.Content = nil
 339 }
 340
 341 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 342         client.StorageClasses = sc
 343 }
 344
 345 type FileWrapper struct {
 346         io.ReadCloser
 347         len int64
 348 }
 349
 350 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 351         return nil, errors.New("not implemented")
 352 }
 353
 354 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 355         return 0, errors.New("not implemented")
 356 }
 357
 358 func (fw FileWrapper) Size() int64 {
 359         return fw.len
 360 }
 361
 362 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 363         return nil, errors.New("not implemented")
 364 }
 365
 366 func (fw FileWrapper) Truncate(int64) error {
 367         return errors.New("not implemented")
 368 }
 369
 370 func (fw FileWrapper) Write([]byte) (int, error) {
 371         return 0, errors.New("not implemented")
 372 }
 373
 374 func (fw FileWrapper) Sync() error {
 375         return errors.New("not implemented")
 376 }
 377
 378 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 379         if filename == hwImageID+".tar" {
 380                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 381                 client.Called = true
 382                 return FileWrapper{rdr, 1321984}, nil
 383         } else if filename == "/file1_in_main.txt" {
 384                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 385                 client.Called = true
 386                 return FileWrapper{rdr, 3}, nil
 387         }
 388         return nil, nil
 389 }
 390
 391 func (s *TestSuite) TestLoadImage(c *C) {
 392         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 393         s.runner.Container.Mounts = map[string]arvados.Mount{
 394                 "/out": {Kind: "tmp", Writable: true},
 395         }
 396         s.runner.Container.OutputPath = "/out"
 397
 398         _, err := s.runner.SetupMounts()
 399         c.Assert(err, IsNil)
 400
 401         imageID, err := s.runner.LoadImage()
 402         c.Check(err, IsNil)
 403         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 404         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 405
 406         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 407         s.executor.imageLoaded = false
 408         s.executor.loaded = ""
 409         s.executor.loadErr = errors.New("bork")
 410         imageID, err = s.runner.LoadImage()
 411         c.Check(err, ErrorMatches, ".*bork")
 412         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 413
 414         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 415         s.executor.imageLoaded = false
 416         s.executor.loaded = ""
 417         s.executor.loadErr = nil
 418         imageID, err = s.runner.LoadImage()
 419         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 420         c.Check(s.executor.loaded, Equals, "")
 421 }
 422
 423 type ArvErrorTestClient struct{}
 424
 425 func (ArvErrorTestClient) Create(resourceType string,
 426         parameters arvadosclient.Dict,
 427         output interface{}) error {
 428         return nil
 429 }
 430
 431 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 432         if method == "GET" && resourceType == "containers" && action == "auth" {
 433                 return nil
 434         }
 435         return errors.New("ArvError")
 436 }
 437
 438 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 439         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 440         return nil, errors.New("ArvError")
 441 }
 442
 443 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 444         return errors.New("ArvError")
 445 }
 446
 447 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 448         return nil
 449 }
 450
 451 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 452         return discoveryMap[key], nil
 453 }
 454
 455 type KeepErrorTestClient struct {
 456         KeepTestClient
 457 }
 458
 459 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 460         return nil, errors.New("KeepError")
 461 }
 462
 463 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 464         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 465 }
 466
 467 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 468         return "", errors.New("KeepError")
 469 }
 470
 471 type KeepReadErrorTestClient struct {
 472         KeepTestClient
 473 }
 474
 475 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 476         return 0, errors.New("KeepError")
 477 }
 478
 479 type ErrorReader struct {
 480         FileWrapper
 481 }
 482
 483 func (ErrorReader) Read(p []byte) (n int, err error) {
 484         return 0, errors.New("ErrorReader")
 485 }
 486
 487 func (ErrorReader) Seek(int64, int) (int64, error) {
 488         return 0, errors.New("ErrorReader")
 489 }
 490
 491 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 492         return ErrorReader{}, nil
 493 }
 494
 495 type ClosableBuffer struct {
 496         bytes.Buffer
 497 }
 498
 499 func (*ClosableBuffer) Close() error {
 500         return nil
 501 }
 502
 503 type TestLogs struct {
 504         Stdout ClosableBuffer
 505         Stderr ClosableBuffer
 506 }
 507
 508 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 509         if logstr == "stdout" {
 510                 return &tl.Stdout, nil
 511         }
 512         if logstr == "stderr" {
 513                 return &tl.Stderr, nil
 514         }
 515         return nil, errors.New("???")
 516 }
 517
 518 func dockerLog(fd byte, msg string) []byte {
 519         by := []byte(msg)
 520         header := make([]byte, 8+len(by))
 521         header[0] = fd
 522         header[7] = byte(len(by))
 523         copy(header[8:], by)
 524         return header
 525 }
 526
 527 func (s *TestSuite) TestRunContainer(c *C) {
 528         s.executor.runFunc = func() {
 529                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 530                 s.executor.exit <- 0
 531         }
 532
 533         var logs TestLogs
 534         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 535         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 536         s.runner.Container.Command = []string{"./hw"}
 537         s.runner.Container.OutputStorageClasses = []string{"default"}
 538
 539         imageID, err := s.runner.LoadImage()
 540         c.Assert(err, IsNil)
 541
 542         err = s.runner.CreateContainer(imageID, nil)
 543         c.Assert(err, IsNil)
 544
 545         err = s.runner.StartContainer()
 546         c.Assert(err, IsNil)
 547
 548         err = s.runner.WaitFinish()
 549         c.Assert(err, IsNil)
 550
 551         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 552         c.Check(logs.Stderr.String(), Equals, "")
 553 }
 554
 555 func (s *TestSuite) TestCommitLogs(c *C) {
 556         api := &ArvTestClient{}
 557         kc := &KeepTestClient{}
 558         defer kc.Close()
 559         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 560         c.Assert(err, IsNil)
 561         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 562
 563         cr.CrunchLog.Print("Hello world!")
 564         cr.CrunchLog.Print("Goodbye")
 565         cr.finalState = "Complete"
 566
 567         err = cr.CommitLogs()
 568         c.Check(err, IsNil)
 569
 570         c.Check(api.Calls, Equals, 2)
 571         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 572         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 573         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 574         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 575 }
 576
 577 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 578         api := &ArvTestClient{}
 579         kc := &KeepTestClient{}
 580         defer kc.Close()
 581         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 582         c.Assert(err, IsNil)
 583
 584         err = cr.UpdateContainerRunning()
 585         c.Check(err, IsNil)
 586
 587         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 588 }
 589
 590 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 591         api := &ArvTestClient{}
 592         kc := &KeepTestClient{}
 593         defer kc.Close()
 594         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 595         c.Assert(err, IsNil)
 596
 597         cr.LogsPDH = new(string)
 598         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 599
 600         cr.ExitCode = new(int)
 601         *cr.ExitCode = 42
 602         cr.finalState = "Complete"
 603
 604         err = cr.UpdateContainerFinal()
 605         c.Check(err, IsNil)
 606
 607         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 608         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 609         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 610 }
 611
 612 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 613         api := &ArvTestClient{}
 614         kc := &KeepTestClient{}
 615         defer kc.Close()
 616         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 617         c.Assert(err, IsNil)
 618         cr.cCancelled = true
 619         cr.finalState = "Cancelled"
 620
 621         err = cr.UpdateContainerFinal()
 622         c.Check(err, IsNil)
 623
 624         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 625         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 626         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 627 }
 628
 629 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 630 // dress rehearsal of the Run() function, starting from a JSON container record.
 631 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 632         err := json.Unmarshal([]byte(record), &s.api.Container)
 633         c.Assert(err, IsNil)
 634         initialState := s.api.Container.State
 635
 636         var sm struct {
 637                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 638         }
 639         err = json.Unmarshal([]byte(record), &sm)
 640         c.Check(err, IsNil)
 641         secretMounts, err := json.Marshal(sm)
 642         c.Assert(err, IsNil)
 643         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 644
 645         s.executor.runFunc = func() {
 646                 fn()
 647                 s.executor.exit <- exitCode
 648         }
 649
 650         s.runner.statInterval = 100 * time.Millisecond
 651         s.runner.containerWatchdogInterval = time.Second
 652
 653         realTemp := c.MkDir()
 654         tempcount := 0
 655         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 656                 tempcount++
 657                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 658                 err := os.Mkdir(d, os.ModePerm)
 659                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 660                         // Test case must have pre-populated the tempdir
 661                         err = nil
 662                 }
 663                 return d, err
 664         }
 665         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 666                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 667         }
 668
 669         if extraMounts != nil && len(extraMounts) > 0 {
 670                 err := s.runner.SetupArvMountPoint("keep")
 671                 c.Check(err, IsNil)
 672
 673                 for _, m := range extraMounts {
 674                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 675                 }
 676         }
 677
 678         err = s.runner.Run()
 679         if s.api.CalledWith("container.state", "Complete") != nil {
 680                 c.Check(err, IsNil)
 681         }
 682         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 683                 c.Check(s.api.WasSetRunning, Equals, true)
 684                 var lastupdate arvadosclient.Dict
 685                 for _, content := range s.api.Content {
 686                         if content["container"] != nil {
 687                                 lastupdate = content["container"].(arvadosclient.Dict)
 688                         }
 689                 }
 690                 if lastupdate["log"] == nil {
 691                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 692                 }
 693         }
 694
 695         if err != nil {
 696                 for k, v := range s.api.Logs {
 697                         c.Log(k)
 698                         c.Log(v.String())
 699                 }
 700         }
 701
 702         return s.api, s.runner, realTemp
 703 }
 704
 705 func (s *TestSuite) TestFullRunHello(c *C) {
 706         s.runner.enableMemoryLimit = true
 707         s.runner.networkMode = "default"
 708         s.fullRunHelper(c, `{
 709     "command": ["echo", "hello world"],
 710     "container_image": "`+arvadostest.DockerImage112PDH+`",
 711     "cwd": ".",
 712     "environment": {"foo":"bar","baz":"waz"},
 713     "mounts": {"/tmp": {"kind": "tmp"} },
 714     "output_path": "/tmp",
 715     "priority": 1,
 716     "runtime_constraints": {"vcpus":1,"ram":1000000},
 717     "state": "Locked",
 718     "output_storage_classes": ["default"]
 719 }`, nil, 0, func() {
 720                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 721                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 722                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 723                 c.Check(s.executor.created.VCPUs, Equals, 1)
 724                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 725                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 726                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 727                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 728                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 729         })
 730
 731         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 732         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 733         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 734         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 735         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 736 }
 737
 738 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 739         var ran bool
 740         s.fullRunHelper(c, `{
 741     "command": ["sleep", "3"],
 742     "container_image": "`+arvadostest.DockerImage112PDH+`",
 743     "cwd": ".",
 744     "environment": {},
 745     "mounts": {"/tmp": {"kind": "tmp"} },
 746     "output_path": "/tmp",
 747     "priority": 1,
 748     "runtime_constraints": {},
 749     "scheduling_parameters":{"max_run_time": 1},
 750     "state": "Running"
 751 }`, nil, 2, func() {
 752                 ran = true
 753         })
 754         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 755         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 756         c.Check(ran, Equals, false)
 757 }
 758
 759 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 760         s.fullRunHelper(c, `{
 761     "command": ["sleep", "3"],
 762     "container_image": "`+arvadostest.DockerImage112PDH+`",
 763     "cwd": ".",
 764     "environment": {},
 765     "mounts": {"/tmp": {"kind": "tmp"} },
 766     "output_path": "/tmp",
 767     "priority": 1,
 768     "runtime_constraints": {},
 769     "scheduling_parameters":{"max_run_time": 1},
 770     "state": "Locked"
 771 }`, nil, 0, func() {
 772                 time.Sleep(3 * time.Second)
 773         })
 774
 775         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 776         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 777 }
 778
 779 func (s *TestSuite) TestContainerWaitFails(c *C) {
 780         s.fullRunHelper(c, `{
 781     "command": ["sleep", "3"],
 782     "container_image": "`+arvadostest.DockerImage112PDH+`",
 783     "cwd": ".",
 784     "mounts": {"/tmp": {"kind": "tmp"} },
 785     "output_path": "/tmp",
 786     "priority": 1,
 787     "state": "Locked"
 788 }`, nil, 0, func() {
 789                 s.executor.waitErr = errors.New("Container is not running")
 790         })
 791
 792         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 793         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 794 }
 795
 796 func (s *TestSuite) TestCrunchstat(c *C) {
 797         s.fullRunHelper(c, `{
 798                 "command": ["sleep", "1"],
 799                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 800                 "cwd": ".",
 801                 "environment": {},
 802                 "mounts": {"/tmp": {"kind": "tmp"} },
 803                 "output_path": "/tmp",
 804                 "priority": 1,
 805                 "runtime_constraints": {},
 806                 "state": "Locked"
 807         }`, nil, 0, func() {
 808                 time.Sleep(time.Second)
 809         })
 810
 811         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 812         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 813
 814         // We didn't actually start a container, so crunchstat didn't
 815         // find accounting files and therefore didn't log any stats.
 816         // It should have logged a "can't find accounting files"
 817         // message after one poll interval, though, so we can confirm
 818         // it's alive:
 819         c.Assert(s.api.Logs["crunchstat"], NotNil)
 820         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 821
 822         // The "files never appeared" log assures us that we called
 823         // (*crunchstat.Reporter)Stop(), and that we set it up with
 824         // the correct container ID "abcde":
 825         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 826 }
 827
 828 func (s *TestSuite) TestNodeInfoLog(c *C) {
 829         os.Setenv("SLURMD_NODENAME", "compute2")
 830         s.fullRunHelper(c, `{
 831                 "command": ["sleep", "1"],
 832                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 833                 "cwd": ".",
 834                 "environment": {},
 835                 "mounts": {"/tmp": {"kind": "tmp"} },
 836                 "output_path": "/tmp",
 837                 "priority": 1,
 838                 "runtime_constraints": {},
 839                 "state": "Locked"
 840         }`, nil, 0,
 841                 func() {
 842                         time.Sleep(time.Second)
 843                 })
 844
 845         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 846         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 847
 848         c.Assert(s.api.Logs["node"], NotNil)
 849         json := s.api.Logs["node"].String()
 850         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 851         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 852         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 853
 854         c.Assert(s.api.Logs["node-info"], NotNil)
 855         json = s.api.Logs["node-info"].String()
 856         c.Check(json, Matches, `(?ms).*Host Information.*`)
 857         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 858         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 859         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 860         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 861 }
 862
 863 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 864         s.fullRunHelper(c, `{
 865                 "command": ["sleep", "1"],
 866                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 867                 "cwd": ".",
 868                 "environment": {},
 869                 "mounts": {"/tmp": {"kind": "tmp"} },
 870                 "output_path": "/tmp",
 871                 "priority": 1,
 872                 "runtime_constraints": {},
 873                 "state": "Locked"
 874         }`, nil, 0,
 875                 func() {
 876                 })
 877
 878         c.Assert(s.api.Logs["crunch-run"], NotNil)
 879         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 880         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 881 }
 882
 883 func (s *TestSuite) TestContainerRecordLog(c *C) {
 884         s.fullRunHelper(c, `{
 885                 "command": ["sleep", "1"],
 886                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 887                 "cwd": ".",
 888                 "environment": {},
 889                 "mounts": {"/tmp": {"kind": "tmp"} },
 890                 "output_path": "/tmp",
 891                 "priority": 1,
 892                 "runtime_constraints": {},
 893                 "state": "Locked"
 894         }`, nil, 0,
 895                 func() {
 896                         time.Sleep(time.Second)
 897                 })
 898
 899         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 900         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 901
 902         c.Assert(s.api.Logs["container"], NotNil)
 903         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 904 }
 905
 906 func (s *TestSuite) TestFullRunStderr(c *C) {
 907         s.fullRunHelper(c, `{
 908     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 909     "container_image": "`+arvadostest.DockerImage112PDH+`",
 910     "cwd": ".",
 911     "environment": {},
 912     "mounts": {"/tmp": {"kind": "tmp"} },
 913     "output_path": "/tmp",
 914     "priority": 1,
 915     "runtime_constraints": {},
 916     "state": "Locked"
 917 }`, nil, 1, func() {
 918                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 919                 fmt.Fprintln(s.executor.created.Stderr, "world")
 920         })
 921
 922         final := s.api.CalledWith("container.state", "Complete")
 923         c.Assert(final, NotNil)
 924         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 925         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 926
 927         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 928         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 929 }
 930
 931 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 932         s.fullRunHelper(c, `{
 933     "command": ["pwd"],
 934     "container_image": "`+arvadostest.DockerImage112PDH+`",
 935     "cwd": ".",
 936     "environment": {},
 937     "mounts": {"/tmp": {"kind": "tmp"} },
 938     "output_path": "/tmp",
 939     "priority": 1,
 940     "runtime_constraints": {},
 941     "state": "Locked"
 942 }`, nil, 0, func() {
 943                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 944         })
 945
 946         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 947         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 948         c.Log(s.api.Logs["stdout"])
 949         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 950 }
 951
 952 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 953         s.fullRunHelper(c, `{
 954     "command": ["pwd"],
 955     "container_image": "`+arvadostest.DockerImage112PDH+`",
 956     "cwd": "/bin",
 957     "environment": {},
 958     "mounts": {"/tmp": {"kind": "tmp"} },
 959     "output_path": "/tmp",
 960     "priority": 1,
 961     "runtime_constraints": {},
 962     "state": "Locked"
 963 }`, nil, 0, func() {
 964                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 965         })
 966
 967         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 968         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 969         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 970 }
 971
 972 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 973         s.fullRunHelper(c, `{
 974     "command": ["pwd"],
 975     "container_image": "`+arvadostest.DockerImage112PDH+`",
 976     "cwd": "/bin",
 977     "environment": {},
 978     "mounts": {"/tmp": {"kind": "tmp"} },
 979     "output_path": "/tmp",
 980     "priority": 1,
 981     "runtime_constraints": {},
 982     "state": "Locked",
 983     "output_storage_classes": ["foo", "bar"]
 984 }`, nil, 0, func() {
 985                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 986         })
 987
 988         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 989         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 990         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 991         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 992         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 993 }
 994
 995 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
 996         s.fullRunHelper(c, `{
 997     "command": ["pwd"],
 998     "container_image": "`+arvadostest.DockerImage112PDH+`",
 999     "cwd": "/bin",
1000     "environment": {},
1001     "mounts": {"/tmp": {"kind": "tmp"} },
1002     "output_path": "/tmp",
1003     "priority": 1,
1004     "runtime_constraints": {"cuda": {"device_count": 2}},
1005     "state": "Locked",
1006     "output_storage_classes": ["foo", "bar"]
1007 }`, nil, 0, func() {
1008                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1009         })
1010         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1011 }
1012
1013 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1014         s.fullRunHelper(c, `{
1015     "command": ["pwd"],
1016     "container_image": "`+arvadostest.DockerImage112PDH+`",
1017     "cwd": "/bin",
1018     "environment": {},
1019     "mounts": {"/tmp": {"kind": "tmp"} },
1020     "output_path": "/tmp",
1021     "priority": 1,
1022     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1023     "state": "Locked",
1024     "output_storage_classes": ["foo", "bar"]
1025 }`, nil, 0, func() {
1026                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1027         })
1028         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1029 }
1030
1031 func (s *TestSuite) TestStopOnSignal(c *C) {
1032         s.executor.runFunc = func() {
1033                 s.executor.created.Stdout.Write([]byte("foo\n"))
1034                 s.runner.SigChan <- syscall.SIGINT
1035         }
1036         s.testStopContainer(c)
1037 }
1038
1039 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1040         s.executor.runFunc = func() {
1041                 s.executor.created.Stdout.Write([]byte("foo\n"))
1042                 s.runner.ArvMountExit <- nil
1043                 close(s.runner.ArvMountExit)
1044         }
1045         s.runner.ArvMountExit = make(chan error)
1046         s.testStopContainer(c)
1047 }
1048
1049 func (s *TestSuite) testStopContainer(c *C) {
1050         record := `{
1051     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1052     "container_image": "` + arvadostest.DockerImage112PDH + `",
1053     "cwd": ".",
1054     "environment": {},
1055     "mounts": {"/tmp": {"kind": "tmp"} },
1056     "output_path": "/tmp",
1057     "priority": 1,
1058     "runtime_constraints": {},
1059     "state": "Locked"
1060 }`
1061
1062         err := json.Unmarshal([]byte(record), &s.api.Container)
1063         c.Assert(err, IsNil)
1064
1065         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1066         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1067                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1068         }
1069
1070         done := make(chan error)
1071         go func() {
1072                 done <- s.runner.Run()
1073         }()
1074         select {
1075         case <-time.After(20 * time.Second):
1076                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1077                 c.Fatal("timed out")
1078         case err = <-done:
1079                 c.Check(err, IsNil)
1080         }
1081         for k, v := range s.api.Logs {
1082                 c.Log(k)
1083                 c.Log(v.String(), "\n")
1084         }
1085
1086         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1087         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1088         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1089 }
1090
1091 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1092         s.fullRunHelper(c, `{
1093     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1094     "container_image": "`+arvadostest.DockerImage112PDH+`",
1095     "cwd": "/bin",
1096     "environment": {"FROBIZ": "bilbo"},
1097     "mounts": {"/tmp": {"kind": "tmp"} },
1098     "output_path": "/tmp",
1099     "priority": 1,
1100     "runtime_constraints": {},
1101     "state": "Locked"
1102 }`, nil, 0, func() {
1103                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1104         })
1105
1106         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1107         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1108         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1109 }
1110
1111 type ArvMountCmdLine struct {
1112         Cmd   []string
1113         token string
1114 }
1115
1116 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1117         am.Cmd = c
1118         am.token = token
1119         return nil, nil
1120 }
1121
1122 func stubCert(temp string) string {
1123         path := temp + "/ca-certificates.crt"
1124         crt, _ := os.Create(path)
1125         crt.Close()
1126         arvadosclient.CertFiles = []string{path}
1127         return path
1128 }
1129
1130 func (s *TestSuite) TestSetupMounts(c *C) {
1131         cr := s.runner
1132         am := &ArvMountCmdLine{}
1133         cr.RunArvMount = am.ArvMountTest
1134         cr.ContainerArvClient = &ArvTestClient{}
1135         cr.ContainerKeepClient = &KeepTestClient{}
1136         cr.Container.OutputStorageClasses = []string{"default"}
1137
1138         realTemp := c.MkDir()
1139         certTemp := c.MkDir()
1140         stubCertPath := stubCert(certTemp)
1141         cr.parentTemp = realTemp
1142
1143         i := 0
1144         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1145                 i++
1146                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1147                 err := os.Mkdir(d, os.ModePerm)
1148                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1149                         // Test case must have pre-populated the tempdir
1150                         err = nil
1151                 }
1152                 return d, err
1153         }
1154
1155         checkEmpty := func() {
1156                 // Should be deleted.
1157                 _, err := os.Stat(realTemp)
1158                 c.Assert(os.IsNotExist(err), Equals, true)
1159
1160                 // Now recreate it for the next test.
1161                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1162         }
1163
1164         {
1165                 i = 0
1166                 cr.ArvMountPoint = ""
1167                 cr.Container.Mounts = make(map[string]arvados.Mount)
1168                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1169                 cr.Container.OutputPath = "/tmp"
1170                 cr.statInterval = 5 * time.Second
1171                 bindmounts, err := cr.SetupMounts()
1172                 c.Check(err, IsNil)
1173                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1174                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1175                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1176                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1177                 os.RemoveAll(cr.ArvMountPoint)
1178                 cr.CleanupDirs()
1179                 checkEmpty()
1180         }
1181
1182         {
1183                 i = 0
1184                 cr.ArvMountPoint = ""
1185                 cr.Container.Mounts = make(map[string]arvados.Mount)
1186                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1187                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1188                 cr.Container.OutputPath = "/out"
1189                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1190
1191                 bindmounts, err := cr.SetupMounts()
1192                 c.Check(err, IsNil)
1193                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1194                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1195                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1196                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1197                 os.RemoveAll(cr.ArvMountPoint)
1198                 cr.CleanupDirs()
1199                 checkEmpty()
1200         }
1201
1202         {
1203                 i = 0
1204                 cr.ArvMountPoint = ""
1205                 cr.Container.Mounts = make(map[string]arvados.Mount)
1206                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1207                 cr.Container.OutputPath = "/tmp"
1208                 cr.Container.RuntimeConstraints.API = true
1209                 cr.Container.OutputStorageClasses = []string{"default"}
1210
1211                 bindmounts, err := cr.SetupMounts()
1212                 c.Check(err, IsNil)
1213                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1214                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1215                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1216                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1217                 os.RemoveAll(cr.ArvMountPoint)
1218                 cr.CleanupDirs()
1219                 checkEmpty()
1220
1221                 cr.Container.RuntimeConstraints.API = false
1222         }
1223
1224         {
1225                 i = 0
1226                 cr.ArvMountPoint = ""
1227                 cr.Container.Mounts = map[string]arvados.Mount{
1228                         "/keeptmp": {Kind: "collection", Writable: true},
1229                 }
1230                 cr.Container.OutputPath = "/keeptmp"
1231
1232                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1233
1234                 bindmounts, err := cr.SetupMounts()
1235                 c.Check(err, IsNil)
1236                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1237                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1238                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1239                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1240                 os.RemoveAll(cr.ArvMountPoint)
1241                 cr.CleanupDirs()
1242                 checkEmpty()
1243         }
1244
1245         {
1246                 i = 0
1247                 cr.ArvMountPoint = ""
1248                 cr.Container.Mounts = map[string]arvados.Mount{
1249                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1250                         "/keepout": {Kind: "collection", Writable: true},
1251                 }
1252                 cr.Container.OutputPath = "/keepout"
1253
1254                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1255                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1256
1257                 bindmounts, err := cr.SetupMounts()
1258                 c.Check(err, IsNil)
1259                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1260                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1261                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1262                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1263                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1264                         "/keepout": {realTemp + "/keep1/tmp0", false},
1265                 })
1266                 os.RemoveAll(cr.ArvMountPoint)
1267                 cr.CleanupDirs()
1268                 checkEmpty()
1269         }
1270
1271         {
1272                 i = 0
1273                 cr.ArvMountPoint = ""
1274                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1275                 cr.Container.Mounts = map[string]arvados.Mount{
1276                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1277                         "/keepout": {Kind: "collection", Writable: true},
1278                 }
1279                 cr.Container.OutputPath = "/keepout"
1280
1281                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1282                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1283
1284                 bindmounts, err := cr.SetupMounts()
1285                 c.Check(err, IsNil)
1286                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1287                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1288                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1289                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1290                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1291                         "/keepout": {realTemp + "/keep1/tmp0", false},
1292                 })
1293                 os.RemoveAll(cr.ArvMountPoint)
1294                 cr.CleanupDirs()
1295                 checkEmpty()
1296         }
1297
1298         for _, test := range []struct {
1299                 in  interface{}
1300                 out string
1301         }{
1302                 {in: "foo", out: `"foo"`},
1303                 {in: nil, out: `null`},
1304                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1305         } {
1306                 i = 0
1307                 cr.ArvMountPoint = ""
1308                 cr.Container.Mounts = map[string]arvados.Mount{
1309                         "/mnt/test.json": {Kind: "json", Content: test.in},
1310                 }
1311                 bindmounts, err := cr.SetupMounts()
1312                 c.Check(err, IsNil)
1313                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1314                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1315                 })
1316                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1317                 c.Check(err, IsNil)
1318                 c.Check(content, DeepEquals, []byte(test.out))
1319                 os.RemoveAll(cr.ArvMountPoint)
1320                 cr.CleanupDirs()
1321                 checkEmpty()
1322         }
1323
1324         for _, test := range []struct {
1325                 in  interface{}
1326                 out string
1327         }{
1328                 {in: "foo", out: `foo`},
1329                 {in: nil, out: "error"},
1330                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1331         } {
1332                 i = 0
1333                 cr.ArvMountPoint = ""
1334                 cr.Container.Mounts = map[string]arvados.Mount{
1335                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1336                 }
1337                 bindmounts, err := cr.SetupMounts()
1338                 if test.out == "error" {
1339                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1340                 } else {
1341                         c.Check(err, IsNil)
1342                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1343                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1344                         })
1345                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1346                         c.Check(err, IsNil)
1347                         c.Check(content, DeepEquals, []byte(test.out))
1348                 }
1349                 os.RemoveAll(cr.ArvMountPoint)
1350                 cr.CleanupDirs()
1351                 checkEmpty()
1352         }
1353
1354         // Read-only mount points are allowed underneath output_dir mount point
1355         {
1356                 i = 0
1357                 cr.ArvMountPoint = ""
1358                 cr.Container.Mounts = make(map[string]arvados.Mount)
1359                 cr.Container.Mounts = map[string]arvados.Mount{
1360                         "/tmp":     {Kind: "tmp"},
1361                         "/tmp/foo": {Kind: "collection"},
1362                 }
1363                 cr.Container.OutputPath = "/tmp"
1364
1365                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1366
1367                 bindmounts, err := cr.SetupMounts()
1368                 c.Check(err, IsNil)
1369                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1370                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1371                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1372                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1373                         "/tmp":     {realTemp + "/tmp2", false},
1374                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1375                 })
1376                 os.RemoveAll(cr.ArvMountPoint)
1377                 cr.CleanupDirs()
1378                 checkEmpty()
1379         }
1380
1381         // Writable mount points copied to output_dir mount point
1382         {
1383                 i = 0
1384                 cr.ArvMountPoint = ""
1385                 cr.Container.Mounts = make(map[string]arvados.Mount)
1386                 cr.Container.Mounts = map[string]arvados.Mount{
1387                         "/tmp": {Kind: "tmp"},
1388                         "/tmp/foo": {Kind: "collection",
1389                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1390                                 Writable:         true},
1391                         "/tmp/bar": {Kind: "collection",
1392                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1393                                 Path:             "baz",
1394                                 Writable:         true},
1395                 }
1396                 cr.Container.OutputPath = "/tmp"
1397
1398                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1399                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1400
1401                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1402                 rf.Write([]byte("bar"))
1403                 rf.Close()
1404
1405                 _, err := cr.SetupMounts()
1406                 c.Check(err, IsNil)
1407                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1408                 c.Check(err, IsNil)
1409                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1410                 c.Check(err, IsNil)
1411                 os.RemoveAll(cr.ArvMountPoint)
1412                 cr.CleanupDirs()
1413                 checkEmpty()
1414         }
1415
1416         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1417         {
1418                 i = 0
1419                 cr.ArvMountPoint = ""
1420                 cr.Container.Mounts = make(map[string]arvados.Mount)
1421                 cr.Container.Mounts = map[string]arvados.Mount{
1422                         "/tmp":     {Kind: "tmp"},
1423                         "/tmp/foo": {Kind: "tmp"},
1424                 }
1425                 cr.Container.OutputPath = "/tmp"
1426
1427                 _, err := cr.SetupMounts()
1428                 c.Check(err, NotNil)
1429                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1430                 os.RemoveAll(cr.ArvMountPoint)
1431                 cr.CleanupDirs()
1432                 checkEmpty()
1433         }
1434
1435         // Only mount point of kind 'collection' is allowed for stdin
1436         {
1437                 i = 0
1438                 cr.ArvMountPoint = ""
1439                 cr.Container.Mounts = make(map[string]arvados.Mount)
1440                 cr.Container.Mounts = map[string]arvados.Mount{
1441                         "stdin": {Kind: "tmp"},
1442                 }
1443
1444                 _, err := cr.SetupMounts()
1445                 c.Check(err, NotNil)
1446                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1447                 os.RemoveAll(cr.ArvMountPoint)
1448                 cr.CleanupDirs()
1449                 checkEmpty()
1450         }
1451
1452         // git_tree mounts
1453         {
1454                 i = 0
1455                 cr.ArvMountPoint = ""
1456                 (*GitMountSuite)(nil).useTestGitServer(c)
1457                 cr.token = arvadostest.ActiveToken
1458                 cr.Container.Mounts = make(map[string]arvados.Mount)
1459                 cr.Container.Mounts = map[string]arvados.Mount{
1460                         "/tip": {
1461                                 Kind:   "git_tree",
1462                                 UUID:   arvadostest.Repository2UUID,
1463                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1464                                 Path:   "/",
1465                         },
1466                         "/non-tip": {
1467                                 Kind:   "git_tree",
1468                                 UUID:   arvadostest.Repository2UUID,
1469                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1470                                 Path:   "/",
1471                         },
1472                 }
1473                 cr.Container.OutputPath = "/tmp"
1474
1475                 bindmounts, err := cr.SetupMounts()
1476                 c.Check(err, IsNil)
1477
1478                 for path, mount := range bindmounts {
1479                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1480                 }
1481
1482                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1483                 c.Check(err, IsNil)
1484                 c.Check(string(data), Equals, "\000\001\002\003")
1485                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1486                 c.Check(err, FitsTypeOf, &os.PathError{})
1487                 c.Check(os.IsNotExist(err), Equals, true)
1488
1489                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1490                 c.Check(err, IsNil)
1491                 c.Check(string(data), Equals, "\000\001\002\003")
1492                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1493                 c.Check(err, IsNil)
1494                 c.Check(string(data), Equals, "testfile\n")
1495
1496                 cr.CleanupDirs()
1497                 checkEmpty()
1498         }
1499 }
1500
1501 func (s *TestSuite) TestStdout(c *C) {
1502         helperRecord := `{
1503                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1504                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1505                 "cwd": "/bin",
1506                 "environment": {"FROBIZ": "bilbo"},
1507                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1508                 "output_path": "/tmp",
1509                 "priority": 1,
1510                 "runtime_constraints": {},
1511                 "state": "Locked"
1512         }`
1513
1514         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1515                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1516         })
1517
1518         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1519         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1520         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1521 }
1522
1523 // Used by the TestStdoutWithWrongPath*()
1524 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1525         err := json.Unmarshal([]byte(record), &s.api.Container)
1526         c.Assert(err, IsNil)
1527         s.executor.runFunc = fn
1528         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1529         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1530                 return s.api, &KeepTestClient{}, nil, nil
1531         }
1532         return s.api, s.runner, s.runner.Run()
1533 }
1534
1535 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1536         _, _, err := s.stdoutErrorRunHelper(c, `{
1537     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1538     "output_path": "/tmp",
1539     "state": "Locked"
1540 }`, func() {})
1541         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1542 }
1543
1544 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1545         _, _, err := s.stdoutErrorRunHelper(c, `{
1546     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1547     "output_path": "/tmp",
1548     "state": "Locked"
1549 }`, func() {})
1550         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1551 }
1552
1553 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1554         _, _, err := s.stdoutErrorRunHelper(c, `{
1555     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1556     "output_path": "/tmp",
1557     "state": "Locked"
1558 }`, func() {})
1559         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1560 }
1561
1562 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1563         s.fullRunHelper(c, `{
1564     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1565     "container_image": "`+arvadostest.DockerImage112PDH+`",
1566     "cwd": "/bin",
1567     "environment": {},
1568     "mounts": {"/tmp": {"kind": "tmp"} },
1569     "output_path": "/tmp",
1570     "priority": 1,
1571     "runtime_constraints": {"API": true},
1572     "state": "Locked"
1573 }`, nil, 0, func() {
1574                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1575                 s.executor.exit <- 3
1576         })
1577         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1578         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1579         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1580 }
1581
1582 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1583         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1584         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1585         s.fullRunHelper(c, `{
1586     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1587     "container_image": "`+arvadostest.DockerImage112PDH+`",
1588     "cwd": "/bin",
1589     "environment": {},
1590     "mounts": {"/tmp": {"kind": "tmp"} },
1591     "output_path": "/tmp",
1592     "priority": 1,
1593     "runtime_constraints": {"API": true},
1594     "state": "Locked"
1595 }`, nil, 0, func() {
1596                 s.api.Container.Output = arvadostest.DockerImage112PDH
1597         })
1598
1599         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1600         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1601         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1602 }
1603
1604 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1605         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1606                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1607                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1608                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1609         }
1610         s.executor.runFunc = func() {
1611                 time.Sleep(time.Second)
1612                 s.executor.exit <- 137
1613         }
1614         record := `{
1615     "command": ["sleep", "1"],
1616     "container_image": "` + arvadostest.DockerImage112PDH + `",
1617     "cwd": "/bin",
1618     "environment": {},
1619     "mounts": {"/tmp": {"kind": "tmp"} },
1620     "output_path": "/tmp",
1621     "priority": 1,
1622     "runtime_constraints": {"API": true},
1623     "state": "Locked"
1624 }`
1625         err := json.Unmarshal([]byte(record), &s.api.Container)
1626         c.Assert(err, IsNil)
1627         err = s.runner.Run()
1628         c.Assert(err, IsNil)
1629         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1630         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1631         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1632         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1633         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1634 }
1635
1636 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1637         helperRecord := `{
1638                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1639                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1640                 "cwd": "/bin",
1641                 "environment": {"FROBIZ": "bilbo"},
1642                 "mounts": {
1643         "/tmp": {"kind": "tmp"},
1644         "/tmp/foo": {"kind": "collection",
1645                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1646                      "exclude_from_output": true
1647         },
1648         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1649     },
1650                 "output_path": "/tmp",
1651                 "priority": 1,
1652                 "runtime_constraints": {},
1653                 "state": "Locked"
1654         }`
1655
1656         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1657
1658         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1659                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1660         })
1661
1662         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1663         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1664         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1665 }
1666
1667 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1668         helperRecord := `{
1669                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1670                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1671                 "cwd": "/bin",
1672                 "environment": {"FROBIZ": "bilbo"},
1673                 "mounts": {
1674         "/tmp": {"kind": "tmp"},
1675         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1676         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1677         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1678         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1679         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1680     },
1681                 "output_path": "/tmp",
1682                 "priority": 1,
1683                 "runtime_constraints": {},
1684                 "state": "Locked"
1685         }`
1686
1687         extraMounts := []string{
1688                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1689                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1690                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1691         }
1692
1693         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1694                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1695         })
1696
1697         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1698                 "/tmp":                   {realtemp + "/tmp1", false},
1699                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1700                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1701                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1702                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1703         })
1704
1705         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1706         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1707         for _, v := range api.Content {
1708                 if v["collection"] != nil {
1709                         c.Check(v["ensure_unique_name"], Equals, true)
1710                         collection := v["collection"].(arvadosclient.Dict)
1711                         if strings.Index(collection["name"].(string), "output") == 0 {
1712                                 manifest := collection["manifest_text"].(string)
1713
1714                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1715 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1716 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1717 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1718 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1719 `)
1720                         }
1721                 }
1722         }
1723 }
1724
1725 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1726         helperRecord := `{
1727                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1728                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1729                 "cwd": "/bin",
1730                 "environment": {"FROBIZ": "bilbo"},
1731                 "mounts": {
1732         "/tmp": {"kind": "tmp"},
1733         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1734         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1735     },
1736                 "output_path": "/tmp",
1737                 "priority": 1,
1738                 "runtime_constraints": {},
1739                 "state": "Locked"
1740         }`
1741
1742         extraMounts := []string{
1743                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1744         }
1745
1746         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1747                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1748         })
1749
1750         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1751         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1752         for _, v := range s.api.Content {
1753                 if v["collection"] != nil {
1754                         collection := v["collection"].(arvadosclient.Dict)
1755                         if strings.Index(collection["name"].(string), "output") == 0 {
1756                                 manifest := collection["manifest_text"].(string)
1757
1758                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1759 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1760 `)
1761                         }
1762                 }
1763         }
1764 }
1765
1766 func (s *TestSuite) TestOutputError(c *C) {
1767         helperRecord := `{
1768                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1769                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1770                 "cwd": "/bin",
1771                 "environment": {"FROBIZ": "bilbo"},
1772                 "mounts": {
1773                         "/tmp": {"kind": "tmp"}
1774                 },
1775                 "output_path": "/tmp",
1776                 "priority": 1,
1777                 "runtime_constraints": {},
1778                 "state": "Locked"
1779         }`
1780         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1781                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1782         })
1783
1784         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1785 }
1786
1787 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1788         helperRecord := `{
1789                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1790                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1791                 "cwd": "/bin",
1792                 "environment": {"FROBIZ": "bilbo"},
1793                 "mounts": {
1794         "/tmp": {"kind": "tmp"},
1795         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1796         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1797     },
1798                 "output_path": "/tmp",
1799                 "priority": 1,
1800                 "runtime_constraints": {},
1801                 "state": "Locked"
1802         }`
1803
1804         extraMounts := []string{
1805                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1806         }
1807
1808         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1809                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1810         })
1811
1812         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1813         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1814         for _, v := range api.Content {
1815                 if v["collection"] != nil {
1816                         collection := v["collection"].(arvadosclient.Dict)
1817                         if strings.Index(collection["name"].(string), "output") == 0 {
1818                                 manifest := collection["manifest_text"].(string)
1819                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1820 `)
1821                         }
1822                 }
1823         }
1824 }
1825
1826 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1827         helperRecord := `{
1828                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1829                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1830                 "cwd": "/bin",
1831                 "environment": {"FROBIZ": "bilbo"},
1832                 "mounts": {
1833         "/tmp": {"kind": "tmp"},
1834         "stdin": {"kind": "json", "content": "foo"},
1835         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1836     },
1837                 "output_path": "/tmp",
1838                 "priority": 1,
1839                 "runtime_constraints": {},
1840                 "state": "Locked"
1841         }`
1842
1843         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1844                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1845         })
1846
1847         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1848         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1849         for _, v := range api.Content {
1850                 if v["collection"] != nil {
1851                         collection := v["collection"].(arvadosclient.Dict)
1852                         if strings.Index(collection["name"].(string), "output") == 0 {
1853                                 manifest := collection["manifest_text"].(string)
1854                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1855 `)
1856                         }
1857                 }
1858         }
1859 }
1860
1861 func (s *TestSuite) TestStderrMount(c *C) {
1862         api, cr, _ := s.fullRunHelper(c, `{
1863     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1864     "container_image": "`+arvadostest.DockerImage112PDH+`",
1865     "cwd": ".",
1866     "environment": {},
1867     "mounts": {"/tmp": {"kind": "tmp"},
1868                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1869                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1870     "output_path": "/tmp",
1871     "priority": 1,
1872     "runtime_constraints": {},
1873     "state": "Locked"
1874 }`, nil, 1, func() {
1875                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1876                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1877         })
1878
1879         final := api.CalledWith("container.state", "Complete")
1880         c.Assert(final, NotNil)
1881         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1882         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1883
1884         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1885 }
1886
1887 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1888         s.api.callraw = true
1889         err := s.runner.fetchContainerRecord()
1890         c.Assert(err, IsNil)
1891         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1892         c.Logf("%#v", s.runner.Container)
1893         c.Check(err, IsNil)
1894         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1895 }
1896
1897 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1898         nextState := ""
1899         for _, setup := range []func(){
1900                 func() {
1901                         c.Log("// waitErr = ocl runtime error")
1902                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1903                         nextState = "Cancelled"
1904                 },
1905                 func() {
1906                         c.Log("// loadErr = cannot connect")
1907                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1908                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1909                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1910                         c.Assert(err, IsNil)
1911                         nextState = "Queued"
1912                 },
1913         } {
1914                 s.SetUpTest(c)
1915                 setup()
1916                 s.fullRunHelper(c, `{
1917     "command": ["echo", "hello world"],
1918     "container_image": "`+arvadostest.DockerImage112PDH+`",
1919     "cwd": ".",
1920     "environment": {},
1921     "mounts": {"/tmp": {"kind": "tmp"} },
1922     "output_path": "/tmp",
1923     "priority": 1,
1924     "runtime_constraints": {},
1925     "state": "Locked"
1926 }`, nil, 0, func() {})
1927                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1928                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1929                 if *brokenNodeHook != "" {
1930                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1931                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1932                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1933                 } else {
1934                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1935                 }
1936         }
1937 }
1938
1939 func (s *TestSuite) TestBadCommand(c *C) {
1940         for _, startError := range []string{
1941                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1942                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1943                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1944         } {
1945                 s.SetUpTest(c)
1946                 s.executor.startErr = errors.New(startError)
1947                 s.fullRunHelper(c, `{
1948     "command": ["echo", "hello world"],
1949     "container_image": "`+arvadostest.DockerImage112PDH+`",
1950     "cwd": ".",
1951     "environment": {},
1952     "mounts": {"/tmp": {"kind": "tmp"} },
1953     "output_path": "/tmp",
1954     "priority": 1,
1955     "runtime_constraints": {},
1956     "state": "Locked"
1957 }`, nil, 0, func() {})
1958                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1959                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1960         }
1961 }
1962
1963 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1964         helperRecord := `{
1965                 "command": ["true"],
1966                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1967                 "cwd": "/bin",
1968                 "mounts": {
1969                     "/tmp": {"kind": "tmp"},
1970                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1971                 },
1972                 "secret_mounts": {
1973                 },
1974                 "output_path": "/tmp",
1975                 "priority": 1,
1976                 "runtime_constraints": {},
1977                 "state": "Locked"
1978         }`
1979
1980         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1981                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1982                 c.Check(err, IsNil)
1983                 c.Check(string(content), Equals, "mypassword")
1984         })
1985
1986         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1987         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1988         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1989         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1990
1991         // under secret mounts, not captured in output
1992         helperRecord = `{
1993                 "command": ["true"],
1994                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1995                 "cwd": "/bin",
1996                 "mounts": {
1997                     "/tmp": {"kind": "tmp"}
1998                 },
1999                 "secret_mounts": {
2000                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2001                 },
2002                 "output_path": "/tmp",
2003                 "priority": 1,
2004                 "runtime_constraints": {},
2005                 "state": "Locked"
2006         }`
2007
2008         s.SetUpTest(c)
2009         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2010                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2011                 c.Check(err, IsNil)
2012                 c.Check(string(content), Equals, "mypassword")
2013         })
2014
2015         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2016         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2017         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2018         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2019
2020         // under secret mounts, output dir is a collection, not captured in output
2021         helperRecord = `{
2022                 "command": ["true"],
2023                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2024                 "cwd": "/bin",
2025                 "mounts": {
2026                     "/tmp": {"kind": "collection", "writable": true}
2027                 },
2028                 "secret_mounts": {
2029                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2030                 },
2031                 "output_path": "/tmp",
2032                 "priority": 1,
2033                 "runtime_constraints": {},
2034                 "state": "Locked"
2035         }`
2036
2037         s.SetUpTest(c)
2038         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2039                 // secret.conf should be provisioned as a separate
2040                 // bind mount, i.e., it should not appear in the
2041                 // (fake) fuse filesystem as viewed from the host.
2042                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2043                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2044                         c.Logf("secret.conf: content %q, err %#v", content, err)
2045                 }
2046                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2047                 c.Check(err, IsNil)
2048         })
2049
2050         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2051         c.Check(err, IsNil)
2052         c.Check(string(content), Equals, "mypassword")
2053         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2054         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2055         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2056         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2057 }
2058
2059 type FakeProcess struct {
2060         cmdLine []string
2061 }
2062
2063 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2064         return fp.cmdLine, nil
2065 }