lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         keepmountTmp             []string
  48         testDispatcherKeepClient KeepTestClient
  49         testContainerKeepClient  KeepTestClient
  50 }
  51
  52 func (s *TestSuite) SetUpTest(c *C) {
  53         *brokenNodeHook = ""
  54         s.client = arvados.NewClientFromEnv()
  55         s.executor = &stubExecutor{}
  56         var err error
  57         s.api = &ArvTestClient{}
  58         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  59         c.Assert(err, IsNil)
  60         s.runner.executor = s.executor
  61         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  62                 return s.api, &s.testContainerKeepClient, s.client, nil
  63         }
  64         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  65                 s.runner.ArvMountPoint = s.keepmount
  66                 for i, opt := range cmd {
  67                         if opt == "--mount-tmp" {
  68                                 err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
  69                                 if err != nil {
  70                                         return nil, err
  71                                 }
  72                                 s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
  73                         }
  74                 }
  75                 return nil, nil
  76         }
  77         s.keepmount = c.MkDir()
  78         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  79         s.keepmountTmp = nil
  80         c.Assert(err, IsNil)
  81         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  82         c.Assert(err, IsNil)
  83         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  84         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  85         c.Assert(err, IsNil)
  86         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  87         c.Assert(err, IsNil)
  88         s.runner.ArvMountPoint = s.keepmount
  89 }
  90
  91 type ArvTestClient struct {
  92         Total   int64
  93         Calls   int
  94         Content []arvadosclient.Dict
  95         arvados.Container
  96         secretMounts []byte
  97         Logs         map[string]*bytes.Buffer
  98         sync.Mutex
  99         WasSetRunning bool
 100         callraw       bool
 101 }
 102
 103 type KeepTestClient struct {
 104         Called         bool
 105         Content        []byte
 106         StorageClasses []string
 107 }
 108
 109 type stubExecutor struct {
 110         imageLoaded bool
 111         loaded      string
 112         loadErr     error
 113         exitCode    int
 114         createErr   error
 115         created     containerSpec
 116         startErr    error
 117         waitSleep   time.Duration
 118         waitErr     error
 119         stopErr     error
 120         stopped     bool
 121         closed      bool
 122         runFunc     func()
 123         exit        chan int
 124 }
 125
 126 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 127         containerClient *arvados.Client) error {
 128         e.loaded = tarball
 129         return e.loadErr
 130 }
 131 func (e *stubExecutor) Runtime() string                 { return "stub" }
 132 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 133 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 134 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 135 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 136 func (e *stubExecutor) Close()                          { e.closed = true }
 137 func (e *stubExecutor) Wait(context.Context) (int, error) {
 138         return <-e.exit, e.waitErr
 139 }
 140
 141 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 142
 143 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 144 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 145 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 146
 147 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 148 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 149
 150 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 151 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 152 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 153 `
 154
 155 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 156
 157 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 158 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 159
 160 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 161 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 162
 163 func (client *ArvTestClient) Create(resourceType string,
 164         parameters arvadosclient.Dict,
 165         output interface{}) error {
 166
 167         client.Mutex.Lock()
 168         defer client.Mutex.Unlock()
 169
 170         client.Calls++
 171         client.Content = append(client.Content, parameters)
 172
 173         if resourceType == "logs" {
 174                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 175                 if client.Logs == nil {
 176                         client.Logs = make(map[string]*bytes.Buffer)
 177                 }
 178                 if client.Logs[et] == nil {
 179                         client.Logs[et] = &bytes.Buffer{}
 180                 }
 181                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 182         }
 183
 184         if resourceType == "collections" && output != nil {
 185                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 186                 outmap := output.(*arvados.Collection)
 187                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 188                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 189         }
 190
 191         return nil
 192 }
 193
 194 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 195         switch {
 196         case method == "GET" && resourceType == "containers" && action == "auth":
 197                 return json.Unmarshal([]byte(`{
 198                         "kind": "arvados#api_client_authorization",
 199                         "uuid": "`+fakeAuthUUID+`",
 200                         "api_token": "`+fakeAuthToken+`"
 201                         }`), output)
 202         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 203                 if client.secretMounts != nil {
 204                         return json.Unmarshal(client.secretMounts, output)
 205                 }
 206                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 207         default:
 208                 return fmt.Errorf("Not found")
 209         }
 210 }
 211
 212 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 213         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 214         var j []byte
 215         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 216                 j = []byte(`{
 217                         "kind": "arvados#nodeList",
 218                         "items": [{
 219                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 220                                 "hostname": "compute2",
 221                                 "properties": {"total_cpu_cores": 16}
 222                         }]}`)
 223         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 224                 if uuid == "" {
 225                         j, err = json.Marshal(map[string]interface{}{
 226                                 "items": []interface{}{client.Container},
 227                                 "kind":  "arvados#nodeList",
 228                         })
 229                 } else {
 230                         j, err = json.Marshal(client.Container)
 231                 }
 232         } else {
 233                 j = []byte(`{
 234                         "command": ["sleep", "1"],
 235                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 236                         "cwd": ".",
 237                         "environment": {},
 238                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 239                         "output_path": "/tmp",
 240                         "priority": 1,
 241                         "runtime_constraints": {}
 242                 }`)
 243         }
 244         return ioutil.NopCloser(bytes.NewReader(j)), err
 245 }
 246
 247 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 248         if resourceType == "collections" {
 249                 if uuid == hwPDH {
 250                         output.(*arvados.Collection).ManifestText = hwManifest
 251                 } else if uuid == otherPDH {
 252                         output.(*arvados.Collection).ManifestText = otherManifest
 253                 } else if uuid == normalizedWithSubdirsPDH {
 254                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 255                 } else if uuid == denormalizedWithSubdirsPDH {
 256                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 257                 }
 258         }
 259         if resourceType == "containers" {
 260                 (*output.(*arvados.Container)) = client.Container
 261         }
 262         return nil
 263 }
 264
 265 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 266         client.Mutex.Lock()
 267         defer client.Mutex.Unlock()
 268         client.Calls++
 269         client.Content = append(client.Content, parameters)
 270         if resourceType == "containers" {
 271                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 272                         client.WasSetRunning = true
 273                 }
 274         } else if resourceType == "collections" {
 275                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 276                 output.(*arvados.Collection).UUID = uuid
 277                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 278         }
 279         return nil
 280 }
 281
 282 var discoveryMap = map[string]interface{}{
 283         "defaultTrashLifetime":               float64(1209600),
 284         "crunchLimitLogBytesPerJob":          float64(67108864),
 285         "crunchLogThrottleBytes":             float64(65536),
 286         "crunchLogThrottlePeriod":            float64(60),
 287         "crunchLogThrottleLines":             float64(1024),
 288         "crunchLogPartialLineThrottlePeriod": float64(5),
 289         "crunchLogBytesPerEvent":             float64(4096),
 290         "crunchLogSecondsBetweenEvents":      float64(1),
 291 }
 292
 293 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 294         return discoveryMap[key], nil
 295 }
 296
 297 // CalledWith returns the parameters from the first API call whose
 298 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 299 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 300 // no call matches, it returns nil.
 301 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 302 call:
 303         for _, content := range client.Content {
 304                 var v interface{} = content
 305                 for _, k := range strings.Split(jpath, ".") {
 306                         if dict, ok := v.(arvadosclient.Dict); !ok {
 307                                 continue call
 308                         } else {
 309                                 v = dict[k]
 310                         }
 311                 }
 312                 if v == expect {
 313                         return content
 314                 }
 315         }
 316         return nil
 317 }
 318
 319 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 320         return locator, nil
 321 }
 322
 323 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 324         client.Content = opts.Data
 325         return arvados.BlockWriteResponse{
 326                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 327         }, nil
 328 }
 329
 330 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 331         return 0, errors.New("not implemented")
 332 }
 333
 334 func (client *KeepTestClient) ClearBlockCache() {
 335 }
 336
 337 func (client *KeepTestClient) Close() {
 338         client.Content = nil
 339 }
 340
 341 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 342         client.StorageClasses = sc
 343 }
 344
 345 type FileWrapper struct {
 346         io.ReadCloser
 347         len int64
 348 }
 349
 350 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 351         return nil, errors.New("not implemented")
 352 }
 353
 354 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 355         return 0, errors.New("not implemented")
 356 }
 357
 358 func (fw FileWrapper) Size() int64 {
 359         return fw.len
 360 }
 361
 362 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 363         return nil, errors.New("not implemented")
 364 }
 365
 366 func (fw FileWrapper) Truncate(int64) error {
 367         return errors.New("not implemented")
 368 }
 369
 370 func (fw FileWrapper) Write([]byte) (int, error) {
 371         return 0, errors.New("not implemented")
 372 }
 373
 374 func (fw FileWrapper) Sync() error {
 375         return errors.New("not implemented")
 376 }
 377
 378 func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
 379         return nil, errors.New("not implemented")
 380 }
 381
 382 func (fw FileWrapper) Splice(*arvados.Subtree) error {
 383         return errors.New("not implemented")
 384 }
 385
 386 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 387         if filename == hwImageID+".tar" {
 388                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 389                 client.Called = true
 390                 return FileWrapper{rdr, 1321984}, nil
 391         } else if filename == "/file1_in_main.txt" {
 392                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 393                 client.Called = true
 394                 return FileWrapper{rdr, 3}, nil
 395         }
 396         return nil, nil
 397 }
 398
 399 func (s *TestSuite) TestLoadImage(c *C) {
 400         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 401         s.runner.Container.Mounts = map[string]arvados.Mount{
 402                 "/out": {Kind: "tmp", Writable: true},
 403         }
 404         s.runner.Container.OutputPath = "/out"
 405
 406         _, err := s.runner.SetupMounts()
 407         c.Assert(err, IsNil)
 408
 409         imageID, err := s.runner.LoadImage()
 410         c.Check(err, IsNil)
 411         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 412         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 413
 414         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 415         s.executor.imageLoaded = false
 416         s.executor.loaded = ""
 417         s.executor.loadErr = errors.New("bork")
 418         imageID, err = s.runner.LoadImage()
 419         c.Check(err, ErrorMatches, ".*bork")
 420         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 421
 422         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 423         s.executor.imageLoaded = false
 424         s.executor.loaded = ""
 425         s.executor.loadErr = nil
 426         imageID, err = s.runner.LoadImage()
 427         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 428         c.Check(s.executor.loaded, Equals, "")
 429 }
 430
 431 type ArvErrorTestClient struct{}
 432
 433 func (ArvErrorTestClient) Create(resourceType string,
 434         parameters arvadosclient.Dict,
 435         output interface{}) error {
 436         return nil
 437 }
 438
 439 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 440         if method == "GET" && resourceType == "containers" && action == "auth" {
 441                 return nil
 442         }
 443         return errors.New("ArvError")
 444 }
 445
 446 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 447         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 448         return nil, errors.New("ArvError")
 449 }
 450
 451 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 452         return errors.New("ArvError")
 453 }
 454
 455 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 456         return nil
 457 }
 458
 459 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 460         return discoveryMap[key], nil
 461 }
 462
 463 type KeepErrorTestClient struct {
 464         KeepTestClient
 465 }
 466
 467 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 468         return nil, errors.New("KeepError")
 469 }
 470
 471 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 472         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 473 }
 474
 475 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 476         return "", errors.New("KeepError")
 477 }
 478
 479 type KeepReadErrorTestClient struct {
 480         KeepTestClient
 481 }
 482
 483 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 484         return 0, errors.New("KeepError")
 485 }
 486
 487 type ErrorReader struct {
 488         FileWrapper
 489 }
 490
 491 func (ErrorReader) Read(p []byte) (n int, err error) {
 492         return 0, errors.New("ErrorReader")
 493 }
 494
 495 func (ErrorReader) Seek(int64, int) (int64, error) {
 496         return 0, errors.New("ErrorReader")
 497 }
 498
 499 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 500         return ErrorReader{}, nil
 501 }
 502
 503 type ClosableBuffer struct {
 504         bytes.Buffer
 505 }
 506
 507 func (*ClosableBuffer) Close() error {
 508         return nil
 509 }
 510
 511 type TestLogs struct {
 512         Stdout ClosableBuffer
 513         Stderr ClosableBuffer
 514 }
 515
 516 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 517         if logstr == "stdout" {
 518                 return &tl.Stdout, nil
 519         }
 520         if logstr == "stderr" {
 521                 return &tl.Stderr, nil
 522         }
 523         return nil, errors.New("???")
 524 }
 525
 526 func dockerLog(fd byte, msg string) []byte {
 527         by := []byte(msg)
 528         header := make([]byte, 8+len(by))
 529         header[0] = fd
 530         header[7] = byte(len(by))
 531         copy(header[8:], by)
 532         return header
 533 }
 534
 535 func (s *TestSuite) TestRunContainer(c *C) {
 536         s.executor.runFunc = func() {
 537                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 538                 s.executor.exit <- 0
 539         }
 540
 541         var logs TestLogs
 542         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 543         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 544         s.runner.Container.Command = []string{"./hw"}
 545         s.runner.Container.OutputStorageClasses = []string{"default"}
 546
 547         imageID, err := s.runner.LoadImage()
 548         c.Assert(err, IsNil)
 549
 550         err = s.runner.CreateContainer(imageID, nil)
 551         c.Assert(err, IsNil)
 552
 553         err = s.runner.StartContainer()
 554         c.Assert(err, IsNil)
 555
 556         err = s.runner.WaitFinish()
 557         c.Assert(err, IsNil)
 558
 559         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 560         c.Check(logs.Stderr.String(), Equals, "")
 561 }
 562
 563 func (s *TestSuite) TestCommitLogs(c *C) {
 564         api := &ArvTestClient{}
 565         kc := &KeepTestClient{}
 566         defer kc.Close()
 567         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 568         c.Assert(err, IsNil)
 569         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 570
 571         cr.CrunchLog.Print("Hello world!")
 572         cr.CrunchLog.Print("Goodbye")
 573         cr.finalState = "Complete"
 574
 575         err = cr.CommitLogs()
 576         c.Check(err, IsNil)
 577
 578         c.Check(api.Calls, Equals, 2)
 579         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 580         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 581         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 582         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 583 }
 584
 585 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 586         api := &ArvTestClient{}
 587         kc := &KeepTestClient{}
 588         defer kc.Close()
 589         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 590         c.Assert(err, IsNil)
 591
 592         err = cr.UpdateContainerRunning()
 593         c.Check(err, IsNil)
 594
 595         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 596 }
 597
 598 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 599         api := &ArvTestClient{}
 600         kc := &KeepTestClient{}
 601         defer kc.Close()
 602         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 603         c.Assert(err, IsNil)
 604
 605         cr.LogsPDH = new(string)
 606         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 607
 608         cr.ExitCode = new(int)
 609         *cr.ExitCode = 42
 610         cr.finalState = "Complete"
 611
 612         err = cr.UpdateContainerFinal()
 613         c.Check(err, IsNil)
 614
 615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 616         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 617         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 618 }
 619
 620 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 621         api := &ArvTestClient{}
 622         kc := &KeepTestClient{}
 623         defer kc.Close()
 624         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 625         c.Assert(err, IsNil)
 626         cr.cCancelled = true
 627         cr.finalState = "Cancelled"
 628
 629         err = cr.UpdateContainerFinal()
 630         c.Check(err, IsNil)
 631
 632         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 633         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 634         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 635 }
 636
 637 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 638 // dress rehearsal of the Run() function, starting from a JSON container record.
 639 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 640         err := json.Unmarshal([]byte(record), &s.api.Container)
 641         c.Assert(err, IsNil)
 642         initialState := s.api.Container.State
 643
 644         var sm struct {
 645                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 646         }
 647         err = json.Unmarshal([]byte(record), &sm)
 648         c.Check(err, IsNil)
 649         secretMounts, err := json.Marshal(sm)
 650         c.Assert(err, IsNil)
 651         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 652
 653         s.executor.runFunc = func() {
 654                 fn()
 655                 s.executor.exit <- exitCode
 656         }
 657
 658         s.runner.statInterval = 100 * time.Millisecond
 659         s.runner.containerWatchdogInterval = time.Second
 660
 661         realTemp := c.MkDir()
 662         tempcount := 0
 663         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 664                 tempcount++
 665                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 666                 err := os.Mkdir(d, os.ModePerm)
 667                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 668                         // Test case must have pre-populated the tempdir
 669                         err = nil
 670                 }
 671                 return d, err
 672         }
 673         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 674                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 675         }
 676
 677         if extraMounts != nil && len(extraMounts) > 0 {
 678                 err := s.runner.SetupArvMountPoint("keep")
 679                 c.Check(err, IsNil)
 680
 681                 for _, m := range extraMounts {
 682                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 683                 }
 684         }
 685
 686         err = s.runner.Run()
 687         if s.api.CalledWith("container.state", "Complete") != nil {
 688                 c.Check(err, IsNil)
 689         }
 690         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 691                 c.Check(s.api.WasSetRunning, Equals, true)
 692                 var lastupdate arvadosclient.Dict
 693                 for _, content := range s.api.Content {
 694                         if content["container"] != nil {
 695                                 lastupdate = content["container"].(arvadosclient.Dict)
 696                         }
 697                 }
 698                 if lastupdate["log"] == nil {
 699                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 700                 }
 701         }
 702
 703         if err != nil {
 704                 for k, v := range s.api.Logs {
 705                         c.Log(k)
 706                         c.Log(v.String())
 707                 }
 708         }
 709
 710         return s.api, s.runner, realTemp
 711 }
 712
 713 func (s *TestSuite) TestFullRunHello(c *C) {
 714         s.runner.enableMemoryLimit = true
 715         s.runner.networkMode = "default"
 716         s.fullRunHelper(c, `{
 717     "command": ["echo", "hello world"],
 718     "container_image": "`+arvadostest.DockerImage112PDH+`",
 719     "cwd": ".",
 720     "environment": {"foo":"bar","baz":"waz"},
 721     "mounts": {"/tmp": {"kind": "tmp"} },
 722     "output_path": "/tmp",
 723     "priority": 1,
 724     "runtime_constraints": {"vcpus":1,"ram":1000000},
 725     "state": "Locked",
 726     "output_storage_classes": ["default"]
 727 }`, nil, 0, func() {
 728                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 729                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 730                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 731                 c.Check(s.executor.created.VCPUs, Equals, 1)
 732                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 733                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 734                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 735                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
 736                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 737         })
 738
 739         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 740         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 741         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 742         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 743         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 744 }
 745
 746 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 747         var ran bool
 748         s.fullRunHelper(c, `{
 749     "command": ["sleep", "3"],
 750     "container_image": "`+arvadostest.DockerImage112PDH+`",
 751     "cwd": ".",
 752     "environment": {},
 753     "mounts": {"/tmp": {"kind": "tmp"} },
 754     "output_path": "/tmp",
 755     "priority": 1,
 756     "runtime_constraints": {},
 757     "scheduling_parameters":{"max_run_time": 1},
 758     "state": "Running"
 759 }`, nil, 2, func() {
 760                 ran = true
 761         })
 762         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 763         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 764         c.Check(ran, Equals, false)
 765 }
 766
 767 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 768         s.fullRunHelper(c, `{
 769     "command": ["sleep", "3"],
 770     "container_image": "`+arvadostest.DockerImage112PDH+`",
 771     "cwd": ".",
 772     "environment": {},
 773     "mounts": {"/tmp": {"kind": "tmp"} },
 774     "output_path": "/tmp",
 775     "priority": 1,
 776     "runtime_constraints": {},
 777     "scheduling_parameters":{"max_run_time": 1},
 778     "state": "Locked"
 779 }`, nil, 0, func() {
 780                 time.Sleep(3 * time.Second)
 781         })
 782
 783         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 784         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 785 }
 786
 787 func (s *TestSuite) TestContainerWaitFails(c *C) {
 788         s.fullRunHelper(c, `{
 789     "command": ["sleep", "3"],
 790     "container_image": "`+arvadostest.DockerImage112PDH+`",
 791     "cwd": ".",
 792     "mounts": {"/tmp": {"kind": "tmp"} },
 793     "output_path": "/tmp",
 794     "priority": 1,
 795     "state": "Locked"
 796 }`, nil, 0, func() {
 797                 s.executor.waitErr = errors.New("Container is not running")
 798         })
 799
 800         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 801         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 802 }
 803
 804 func (s *TestSuite) TestCrunchstat(c *C) {
 805         s.fullRunHelper(c, `{
 806                 "command": ["sleep", "1"],
 807                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 808                 "cwd": ".",
 809                 "environment": {},
 810                 "mounts": {"/tmp": {"kind": "tmp"} },
 811                 "output_path": "/tmp",
 812                 "priority": 1,
 813                 "runtime_constraints": {},
 814                 "state": "Locked"
 815         }`, nil, 0, func() {
 816                 time.Sleep(time.Second)
 817         })
 818
 819         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 820         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 821
 822         // We didn't actually start a container, so crunchstat didn't
 823         // find accounting files and therefore didn't log any stats.
 824         // It should have logged a "can't find accounting files"
 825         // message after one poll interval, though, so we can confirm
 826         // it's alive:
 827         c.Assert(s.api.Logs["crunchstat"], NotNil)
 828         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 829
 830         // The "files never appeared" log assures us that we called
 831         // (*crunchstat.Reporter)Stop(), and that we set it up with
 832         // the correct container ID "abcde":
 833         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 834 }
 835
 836 func (s *TestSuite) TestNodeInfoLog(c *C) {
 837         os.Setenv("SLURMD_NODENAME", "compute2")
 838         s.fullRunHelper(c, `{
 839                 "command": ["sleep", "1"],
 840                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 841                 "cwd": ".",
 842                 "environment": {},
 843                 "mounts": {"/tmp": {"kind": "tmp"} },
 844                 "output_path": "/tmp",
 845                 "priority": 1,
 846                 "runtime_constraints": {},
 847                 "state": "Locked"
 848         }`, nil, 0,
 849                 func() {
 850                         time.Sleep(time.Second)
 851                 })
 852
 853         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 854         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 855
 856         c.Assert(s.api.Logs["node"], NotNil)
 857         json := s.api.Logs["node"].String()
 858         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 859         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 860         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 861
 862         c.Assert(s.api.Logs["node-info"], NotNil)
 863         json = s.api.Logs["node-info"].String()
 864         c.Check(json, Matches, `(?ms).*Host Information.*`)
 865         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 866         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 867         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 868         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 869 }
 870
 871 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 872         s.fullRunHelper(c, `{
 873                 "command": ["sleep", "1"],
 874                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 875                 "cwd": ".",
 876                 "environment": {},
 877                 "mounts": {"/tmp": {"kind": "tmp"} },
 878                 "output_path": "/tmp",
 879                 "priority": 1,
 880                 "runtime_constraints": {},
 881                 "state": "Locked"
 882         }`, nil, 0,
 883                 func() {
 884                 })
 885
 886         c.Assert(s.api.Logs["crunch-run"], NotNil)
 887         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
 888         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 889 }
 890
 891 func (s *TestSuite) TestContainerRecordLog(c *C) {
 892         s.fullRunHelper(c, `{
 893                 "command": ["sleep", "1"],
 894                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 895                 "cwd": ".",
 896                 "environment": {},
 897                 "mounts": {"/tmp": {"kind": "tmp"} },
 898                 "output_path": "/tmp",
 899                 "priority": 1,
 900                 "runtime_constraints": {},
 901                 "state": "Locked"
 902         }`, nil, 0,
 903                 func() {
 904                         time.Sleep(time.Second)
 905                 })
 906
 907         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 908         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 909
 910         c.Assert(s.api.Logs["container"], NotNil)
 911         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 912 }
 913
 914 func (s *TestSuite) TestFullRunStderr(c *C) {
 915         s.fullRunHelper(c, `{
 916     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 917     "container_image": "`+arvadostest.DockerImage112PDH+`",
 918     "cwd": ".",
 919     "environment": {},
 920     "mounts": {"/tmp": {"kind": "tmp"} },
 921     "output_path": "/tmp",
 922     "priority": 1,
 923     "runtime_constraints": {},
 924     "state": "Locked"
 925 }`, nil, 1, func() {
 926                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 927                 fmt.Fprintln(s.executor.created.Stderr, "world")
 928         })
 929
 930         final := s.api.CalledWith("container.state", "Complete")
 931         c.Assert(final, NotNil)
 932         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 933         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 934
 935         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 936         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 937 }
 938
 939 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 940         s.fullRunHelper(c, `{
 941     "command": ["pwd"],
 942     "container_image": "`+arvadostest.DockerImage112PDH+`",
 943     "cwd": ".",
 944     "environment": {},
 945     "mounts": {"/tmp": {"kind": "tmp"} },
 946     "output_path": "/tmp",
 947     "priority": 1,
 948     "runtime_constraints": {},
 949     "state": "Locked"
 950 }`, nil, 0, func() {
 951                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 952         })
 953
 954         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 955         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 956         c.Log(s.api.Logs["stdout"])
 957         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 958 }
 959
 960 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 961         s.fullRunHelper(c, `{
 962     "command": ["pwd"],
 963     "container_image": "`+arvadostest.DockerImage112PDH+`",
 964     "cwd": "/bin",
 965     "environment": {},
 966     "mounts": {"/tmp": {"kind": "tmp"} },
 967     "output_path": "/tmp",
 968     "priority": 1,
 969     "runtime_constraints": {},
 970     "state": "Locked"
 971 }`, nil, 0, func() {
 972                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 973         })
 974
 975         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 976         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 977         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 978 }
 979
 980 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 981         s.fullRunHelper(c, `{
 982     "command": ["pwd"],
 983     "container_image": "`+arvadostest.DockerImage112PDH+`",
 984     "cwd": "/bin",
 985     "environment": {},
 986     "mounts": {"/tmp": {"kind": "tmp"} },
 987     "output_path": "/tmp",
 988     "priority": 1,
 989     "runtime_constraints": {},
 990     "state": "Locked",
 991     "output_storage_classes": ["foo", "bar"]
 992 }`, nil, 0, func() {
 993                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 994         })
 995
 996         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 997         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 998         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 999         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1000         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
1001 }
1002
1003 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
1004         s.fullRunHelper(c, `{
1005     "command": ["pwd"],
1006     "container_image": "`+arvadostest.DockerImage112PDH+`",
1007     "cwd": "/bin",
1008     "environment": {},
1009     "mounts": {"/tmp": {"kind": "tmp"} },
1010     "output_path": "/tmp",
1011     "priority": 1,
1012     "runtime_constraints": {"cuda": {"device_count": 2}},
1013     "state": "Locked",
1014     "output_storage_classes": ["foo", "bar"]
1015 }`, nil, 0, func() {
1016                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1017         })
1018         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1019 }
1020
1021 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1022         s.fullRunHelper(c, `{
1023     "command": ["pwd"],
1024     "container_image": "`+arvadostest.DockerImage112PDH+`",
1025     "cwd": "/bin",
1026     "environment": {},
1027     "mounts": {"/tmp": {"kind": "tmp"} },
1028     "output_path": "/tmp",
1029     "priority": 1,
1030     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1031     "state": "Locked",
1032     "output_storage_classes": ["foo", "bar"]
1033 }`, nil, 0, func() {
1034                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1035         })
1036         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1037 }
1038
1039 func (s *TestSuite) TestStopOnSignal(c *C) {
1040         s.executor.runFunc = func() {
1041                 s.executor.created.Stdout.Write([]byte("foo\n"))
1042                 s.runner.SigChan <- syscall.SIGINT
1043         }
1044         s.testStopContainer(c)
1045 }
1046
1047 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1048         s.executor.runFunc = func() {
1049                 s.executor.created.Stdout.Write([]byte("foo\n"))
1050                 s.runner.ArvMountExit <- nil
1051                 close(s.runner.ArvMountExit)
1052         }
1053         s.runner.ArvMountExit = make(chan error)
1054         s.testStopContainer(c)
1055 }
1056
1057 func (s *TestSuite) testStopContainer(c *C) {
1058         record := `{
1059     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1060     "container_image": "` + arvadostest.DockerImage112PDH + `",
1061     "cwd": ".",
1062     "environment": {},
1063     "mounts": {"/tmp": {"kind": "tmp"} },
1064     "output_path": "/tmp",
1065     "priority": 1,
1066     "runtime_constraints": {},
1067     "state": "Locked"
1068 }`
1069
1070         err := json.Unmarshal([]byte(record), &s.api.Container)
1071         c.Assert(err, IsNil)
1072
1073         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1074         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1075                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1076         }
1077
1078         done := make(chan error)
1079         go func() {
1080                 done <- s.runner.Run()
1081         }()
1082         select {
1083         case <-time.After(20 * time.Second):
1084                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1085                 c.Fatal("timed out")
1086         case err = <-done:
1087                 c.Check(err, IsNil)
1088         }
1089         for k, v := range s.api.Logs {
1090                 c.Log(k)
1091                 c.Log(v.String(), "\n")
1092         }
1093
1094         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1095         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1096         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1097 }
1098
1099 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1100         s.fullRunHelper(c, `{
1101     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1102     "container_image": "`+arvadostest.DockerImage112PDH+`",
1103     "cwd": "/bin",
1104     "environment": {"FROBIZ": "bilbo"},
1105     "mounts": {"/tmp": {"kind": "tmp"} },
1106     "output_path": "/tmp",
1107     "priority": 1,
1108     "runtime_constraints": {},
1109     "state": "Locked"
1110 }`, nil, 0, func() {
1111                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1112         })
1113
1114         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1115         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1116         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1117 }
1118
1119 type ArvMountCmdLine struct {
1120         Cmd   []string
1121         token string
1122 }
1123
1124 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1125         am.Cmd = c
1126         am.token = token
1127         return nil, nil
1128 }
1129
1130 func stubCert(temp string) string {
1131         path := temp + "/ca-certificates.crt"
1132         crt, _ := os.Create(path)
1133         crt.Close()
1134         arvadosclient.CertFiles = []string{path}
1135         return path
1136 }
1137
1138 func (s *TestSuite) TestSetupMounts(c *C) {
1139         cr := s.runner
1140         am := &ArvMountCmdLine{}
1141         cr.RunArvMount = am.ArvMountTest
1142         cr.ContainerArvClient = &ArvTestClient{}
1143         cr.ContainerKeepClient = &KeepTestClient{}
1144         cr.Container.OutputStorageClasses = []string{"default"}
1145
1146         realTemp := c.MkDir()
1147         certTemp := c.MkDir()
1148         stubCertPath := stubCert(certTemp)
1149         cr.parentTemp = realTemp
1150
1151         i := 0
1152         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1153                 i++
1154                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1155                 err := os.Mkdir(d, os.ModePerm)
1156                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1157                         // Test case must have pre-populated the tempdir
1158                         err = nil
1159                 }
1160                 return d, err
1161         }
1162
1163         checkEmpty := func() {
1164                 // Should be deleted.
1165                 _, err := os.Stat(realTemp)
1166                 c.Assert(os.IsNotExist(err), Equals, true)
1167
1168                 // Now recreate it for the next test.
1169                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1170         }
1171
1172         {
1173                 i = 0
1174                 cr.ArvMountPoint = ""
1175                 cr.Container.Mounts = make(map[string]arvados.Mount)
1176                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1177                 cr.Container.OutputPath = "/tmp"
1178                 cr.statInterval = 5 * time.Second
1179                 bindmounts, err := cr.SetupMounts()
1180                 c.Check(err, IsNil)
1181                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1182                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1183                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1184                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1185                 os.RemoveAll(cr.ArvMountPoint)
1186                 cr.CleanupDirs()
1187                 checkEmpty()
1188         }
1189
1190         {
1191                 i = 0
1192                 cr.ArvMountPoint = ""
1193                 cr.Container.Mounts = make(map[string]arvados.Mount)
1194                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1195                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1196                 cr.Container.OutputPath = "/out"
1197                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1198
1199                 bindmounts, err := cr.SetupMounts()
1200                 c.Check(err, IsNil)
1201                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1202                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1203                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1204                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1205                 os.RemoveAll(cr.ArvMountPoint)
1206                 cr.CleanupDirs()
1207                 checkEmpty()
1208         }
1209
1210         {
1211                 i = 0
1212                 cr.ArvMountPoint = ""
1213                 cr.Container.Mounts = make(map[string]arvados.Mount)
1214                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1215                 cr.Container.OutputPath = "/tmp"
1216                 cr.Container.RuntimeConstraints.API = true
1217                 cr.Container.OutputStorageClasses = []string{"default"}
1218
1219                 bindmounts, err := cr.SetupMounts()
1220                 c.Check(err, IsNil)
1221                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1222                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1223                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1224                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1225                 os.RemoveAll(cr.ArvMountPoint)
1226                 cr.CleanupDirs()
1227                 checkEmpty()
1228
1229                 cr.Container.RuntimeConstraints.API = false
1230         }
1231
1232         {
1233                 i = 0
1234                 cr.ArvMountPoint = ""
1235                 cr.Container.Mounts = map[string]arvados.Mount{
1236                         "/keeptmp": {Kind: "collection", Writable: true},
1237                 }
1238                 cr.Container.OutputPath = "/keeptmp"
1239
1240                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1241
1242                 bindmounts, err := cr.SetupMounts()
1243                 c.Check(err, IsNil)
1244                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1245                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1246                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1247                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1248                 os.RemoveAll(cr.ArvMountPoint)
1249                 cr.CleanupDirs()
1250                 checkEmpty()
1251         }
1252
1253         {
1254                 i = 0
1255                 cr.ArvMountPoint = ""
1256                 cr.Container.Mounts = map[string]arvados.Mount{
1257                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1258                         "/keepout": {Kind: "collection", Writable: true},
1259                 }
1260                 cr.Container.OutputPath = "/keepout"
1261
1262                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1263                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1264
1265                 bindmounts, err := cr.SetupMounts()
1266                 c.Check(err, IsNil)
1267                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1268                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1269                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1270                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1271                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1272                         "/keepout": {realTemp + "/keep1/tmp0", false},
1273                 })
1274                 os.RemoveAll(cr.ArvMountPoint)
1275                 cr.CleanupDirs()
1276                 checkEmpty()
1277         }
1278
1279         {
1280                 i = 0
1281                 cr.ArvMountPoint = ""
1282                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1283                 cr.Container.Mounts = map[string]arvados.Mount{
1284                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1285                         "/keepout": {Kind: "collection", Writable: true},
1286                 }
1287                 cr.Container.OutputPath = "/keepout"
1288
1289                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1290                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1291
1292                 bindmounts, err := cr.SetupMounts()
1293                 c.Check(err, IsNil)
1294                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1295                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1296                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1297                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1298                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1299                         "/keepout": {realTemp + "/keep1/tmp0", false},
1300                 })
1301                 os.RemoveAll(cr.ArvMountPoint)
1302                 cr.CleanupDirs()
1303                 checkEmpty()
1304         }
1305
1306         for _, test := range []struct {
1307                 in  interface{}
1308                 out string
1309         }{
1310                 {in: "foo", out: `"foo"`},
1311                 {in: nil, out: `null`},
1312                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1313         } {
1314                 i = 0
1315                 cr.ArvMountPoint = ""
1316                 cr.Container.Mounts = map[string]arvados.Mount{
1317                         "/mnt/test.json": {Kind: "json", Content: test.in},
1318                 }
1319                 bindmounts, err := cr.SetupMounts()
1320                 c.Check(err, IsNil)
1321                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1322                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1323                 })
1324                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1325                 c.Check(err, IsNil)
1326                 c.Check(content, DeepEquals, []byte(test.out))
1327                 os.RemoveAll(cr.ArvMountPoint)
1328                 cr.CleanupDirs()
1329                 checkEmpty()
1330         }
1331
1332         for _, test := range []struct {
1333                 in  interface{}
1334                 out string
1335         }{
1336                 {in: "foo", out: `foo`},
1337                 {in: nil, out: "error"},
1338                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1339         } {
1340                 i = 0
1341                 cr.ArvMountPoint = ""
1342                 cr.Container.Mounts = map[string]arvados.Mount{
1343                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1344                 }
1345                 bindmounts, err := cr.SetupMounts()
1346                 if test.out == "error" {
1347                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1348                 } else {
1349                         c.Check(err, IsNil)
1350                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1351                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1352                         })
1353                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1354                         c.Check(err, IsNil)
1355                         c.Check(content, DeepEquals, []byte(test.out))
1356                 }
1357                 os.RemoveAll(cr.ArvMountPoint)
1358                 cr.CleanupDirs()
1359                 checkEmpty()
1360         }
1361
1362         // Read-only mount points are allowed underneath output_dir mount point
1363         {
1364                 i = 0
1365                 cr.ArvMountPoint = ""
1366                 cr.Container.Mounts = make(map[string]arvados.Mount)
1367                 cr.Container.Mounts = map[string]arvados.Mount{
1368                         "/tmp":     {Kind: "tmp"},
1369                         "/tmp/foo": {Kind: "collection"},
1370                 }
1371                 cr.Container.OutputPath = "/tmp"
1372
1373                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1374
1375                 bindmounts, err := cr.SetupMounts()
1376                 c.Check(err, IsNil)
1377                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1378                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1379                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1380                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1381                         "/tmp":     {realTemp + "/tmp2", false},
1382                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1383                 })
1384                 os.RemoveAll(cr.ArvMountPoint)
1385                 cr.CleanupDirs()
1386                 checkEmpty()
1387         }
1388
1389         // Writable mount points copied to output_dir mount point
1390         {
1391                 i = 0
1392                 cr.ArvMountPoint = ""
1393                 cr.Container.Mounts = make(map[string]arvados.Mount)
1394                 cr.Container.Mounts = map[string]arvados.Mount{
1395                         "/tmp": {Kind: "tmp"},
1396                         "/tmp/foo": {Kind: "collection",
1397                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1398                                 Writable:         true},
1399                         "/tmp/bar": {Kind: "collection",
1400                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1401                                 Path:             "baz",
1402                                 Writable:         true},
1403                 }
1404                 cr.Container.OutputPath = "/tmp"
1405
1406                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1407                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1408
1409                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1410                 rf.Write([]byte("bar"))
1411                 rf.Close()
1412
1413                 _, err := cr.SetupMounts()
1414                 c.Check(err, IsNil)
1415                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1416                 c.Check(err, IsNil)
1417                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1418                 c.Check(err, IsNil)
1419                 os.RemoveAll(cr.ArvMountPoint)
1420                 cr.CleanupDirs()
1421                 checkEmpty()
1422         }
1423
1424         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1425         {
1426                 i = 0
1427                 cr.ArvMountPoint = ""
1428                 cr.Container.Mounts = make(map[string]arvados.Mount)
1429                 cr.Container.Mounts = map[string]arvados.Mount{
1430                         "/tmp":     {Kind: "tmp"},
1431                         "/tmp/foo": {Kind: "tmp"},
1432                 }
1433                 cr.Container.OutputPath = "/tmp"
1434
1435                 _, err := cr.SetupMounts()
1436                 c.Check(err, NotNil)
1437                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1438                 os.RemoveAll(cr.ArvMountPoint)
1439                 cr.CleanupDirs()
1440                 checkEmpty()
1441         }
1442
1443         // Only mount point of kind 'collection' is allowed for stdin
1444         {
1445                 i = 0
1446                 cr.ArvMountPoint = ""
1447                 cr.Container.Mounts = make(map[string]arvados.Mount)
1448                 cr.Container.Mounts = map[string]arvados.Mount{
1449                         "stdin": {Kind: "tmp"},
1450                 }
1451
1452                 _, err := cr.SetupMounts()
1453                 c.Check(err, NotNil)
1454                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1455                 os.RemoveAll(cr.ArvMountPoint)
1456                 cr.CleanupDirs()
1457                 checkEmpty()
1458         }
1459
1460         // git_tree mounts
1461         {
1462                 i = 0
1463                 cr.ArvMountPoint = ""
1464                 (*GitMountSuite)(nil).useTestGitServer(c)
1465                 cr.token = arvadostest.ActiveToken
1466                 cr.Container.Mounts = make(map[string]arvados.Mount)
1467                 cr.Container.Mounts = map[string]arvados.Mount{
1468                         "/tip": {
1469                                 Kind:   "git_tree",
1470                                 UUID:   arvadostest.Repository2UUID,
1471                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1472                                 Path:   "/",
1473                         },
1474                         "/non-tip": {
1475                                 Kind:   "git_tree",
1476                                 UUID:   arvadostest.Repository2UUID,
1477                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1478                                 Path:   "/",
1479                         },
1480                 }
1481                 cr.Container.OutputPath = "/tmp"
1482
1483                 bindmounts, err := cr.SetupMounts()
1484                 c.Check(err, IsNil)
1485
1486                 for path, mount := range bindmounts {
1487                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1488                 }
1489
1490                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1491                 c.Check(err, IsNil)
1492                 c.Check(string(data), Equals, "\000\001\002\003")
1493                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1494                 c.Check(err, FitsTypeOf, &os.PathError{})
1495                 c.Check(os.IsNotExist(err), Equals, true)
1496
1497                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1498                 c.Check(err, IsNil)
1499                 c.Check(string(data), Equals, "\000\001\002\003")
1500                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1501                 c.Check(err, IsNil)
1502                 c.Check(string(data), Equals, "testfile\n")
1503
1504                 cr.CleanupDirs()
1505                 checkEmpty()
1506         }
1507 }
1508
1509 func (s *TestSuite) TestStdout(c *C) {
1510         helperRecord := `{
1511                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1512                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1513                 "cwd": "/bin",
1514                 "environment": {"FROBIZ": "bilbo"},
1515                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1516                 "output_path": "/tmp",
1517                 "priority": 1,
1518                 "runtime_constraints": {},
1519                 "state": "Locked"
1520         }`
1521
1522         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1523                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1524         })
1525
1526         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1527         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1528         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1529 }
1530
1531 // Used by the TestStdoutWithWrongPath*()
1532 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1533         err := json.Unmarshal([]byte(record), &s.api.Container)
1534         c.Assert(err, IsNil)
1535         s.executor.runFunc = fn
1536         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1537         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1538                 return s.api, &KeepTestClient{}, nil, nil
1539         }
1540         return s.api, s.runner, s.runner.Run()
1541 }
1542
1543 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1544         _, _, err := s.stdoutErrorRunHelper(c, `{
1545     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1546     "output_path": "/tmp",
1547     "state": "Locked"
1548 }`, func() {})
1549         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1550 }
1551
1552 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1553         _, _, err := s.stdoutErrorRunHelper(c, `{
1554     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1555     "output_path": "/tmp",
1556     "state": "Locked"
1557 }`, func() {})
1558         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1559 }
1560
1561 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1562         _, _, err := s.stdoutErrorRunHelper(c, `{
1563     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1564     "output_path": "/tmp",
1565     "state": "Locked"
1566 }`, func() {})
1567         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1568 }
1569
1570 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1571         s.fullRunHelper(c, `{
1572     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1573     "container_image": "`+arvadostest.DockerImage112PDH+`",
1574     "cwd": "/bin",
1575     "environment": {},
1576     "mounts": {"/tmp": {"kind": "tmp"} },
1577     "output_path": "/tmp",
1578     "priority": 1,
1579     "runtime_constraints": {"API": true},
1580     "state": "Locked"
1581 }`, nil, 0, func() {
1582                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1583                 s.executor.exit <- 3
1584         })
1585         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1586         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1587         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1588 }
1589
1590 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1591         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1592         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1593         s.fullRunHelper(c, `{
1594     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1595     "container_image": "`+arvadostest.DockerImage112PDH+`",
1596     "cwd": "/bin",
1597     "environment": {},
1598     "mounts": {"/tmp": {"kind": "tmp"} },
1599     "output_path": "/tmp",
1600     "priority": 1,
1601     "runtime_constraints": {"API": true},
1602     "state": "Locked"
1603 }`, nil, 0, func() {
1604                 s.api.Container.Output = arvadostest.DockerImage112PDH
1605         })
1606
1607         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1608         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1609         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1610 }
1611
1612 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1613         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1614                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1615                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1616                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1617         }
1618         s.executor.runFunc = func() {
1619                 time.Sleep(time.Second)
1620                 s.executor.exit <- 137
1621         }
1622         record := `{
1623     "command": ["sleep", "1"],
1624     "container_image": "` + arvadostest.DockerImage112PDH + `",
1625     "cwd": "/bin",
1626     "environment": {},
1627     "mounts": {"/tmp": {"kind": "tmp"} },
1628     "output_path": "/tmp",
1629     "priority": 1,
1630     "runtime_constraints": {"API": true},
1631     "state": "Locked"
1632 }`
1633         err := json.Unmarshal([]byte(record), &s.api.Container)
1634         c.Assert(err, IsNil)
1635         err = s.runner.Run()
1636         c.Assert(err, IsNil)
1637         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1638         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1639         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1640         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1641         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1642 }
1643
1644 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1645         helperRecord := `{
1646                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1647                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1648                 "cwd": "/bin",
1649                 "environment": {"FROBIZ": "bilbo"},
1650                 "mounts": {
1651         "/tmp": {"kind": "tmp"},
1652         "/tmp/foo": {"kind": "collection",
1653                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1654                      "exclude_from_output": true
1655         },
1656         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1657     },
1658                 "output_path": "/tmp",
1659                 "priority": 1,
1660                 "runtime_constraints": {},
1661                 "state": "Locked"
1662         }`
1663
1664         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1665
1666         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1667                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1668         })
1669
1670         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1671         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1672         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1673 }
1674
1675 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1676         helperRecord := `{
1677                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1678                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1679                 "cwd": "/bin",
1680                 "environment": {"FROBIZ": "bilbo"},
1681                 "mounts": {
1682         "/tmp": {"kind": "tmp"},
1683         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1684         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1685         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1686         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1687         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1688     },
1689                 "output_path": "/tmp",
1690                 "priority": 1,
1691                 "runtime_constraints": {},
1692                 "state": "Locked"
1693         }`
1694
1695         extraMounts := []string{
1696                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1697                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1698                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1699         }
1700
1701         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1702                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1703         })
1704
1705         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1706                 "/tmp":                   {realtemp + "/tmp1", false},
1707                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1708                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1709                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1710                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1711         })
1712
1713         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1714         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1715         for _, v := range api.Content {
1716                 if v["collection"] != nil {
1717                         c.Check(v["ensure_unique_name"], Equals, true)
1718                         collection := v["collection"].(arvadosclient.Dict)
1719                         if strings.Index(collection["name"].(string), "output") == 0 {
1720                                 manifest := collection["manifest_text"].(string)
1721
1722                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1723 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1724 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1725 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1726 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1727 `)
1728                         }
1729                 }
1730         }
1731 }
1732
1733 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1734         helperRecord := `{
1735                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1736                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1737                 "cwd": "/bin",
1738                 "environment": {"FROBIZ": "bilbo"},
1739                 "mounts": {
1740         "/tmp": {"kind": "tmp"},
1741         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1742         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1743     },
1744                 "output_path": "/tmp",
1745                 "priority": 1,
1746                 "runtime_constraints": {},
1747                 "state": "Locked"
1748         }`
1749
1750         extraMounts := []string{
1751                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1752         }
1753
1754         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1755                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1756         })
1757
1758         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1759         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1760         for _, v := range s.api.Content {
1761                 if v["collection"] != nil {
1762                         collection := v["collection"].(arvadosclient.Dict)
1763                         if strings.Index(collection["name"].(string), "output") == 0 {
1764                                 manifest := collection["manifest_text"].(string)
1765
1766                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1767 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1768 `)
1769                         }
1770                 }
1771         }
1772 }
1773
1774 func (s *TestSuite) TestOutputError(c *C) {
1775         helperRecord := `{
1776                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1777                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1778                 "cwd": "/bin",
1779                 "environment": {"FROBIZ": "bilbo"},
1780                 "mounts": {
1781                         "/tmp": {"kind": "tmp"}
1782                 },
1783                 "output_path": "/tmp",
1784                 "priority": 1,
1785                 "runtime_constraints": {},
1786                 "state": "Locked"
1787         }`
1788         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1789                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1790         })
1791
1792         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1793 }
1794
1795 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1796         helperRecord := `{
1797                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1798                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1799                 "cwd": "/bin",
1800                 "environment": {"FROBIZ": "bilbo"},
1801                 "mounts": {
1802         "/tmp": {"kind": "tmp"},
1803         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1804         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1805     },
1806                 "output_path": "/tmp",
1807                 "priority": 1,
1808                 "runtime_constraints": {},
1809                 "state": "Locked"
1810         }`
1811
1812         extraMounts := []string{
1813                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1814         }
1815
1816         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1817                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1818         })
1819
1820         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1821         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1822         for _, v := range api.Content {
1823                 if v["collection"] != nil {
1824                         collection := v["collection"].(arvadosclient.Dict)
1825                         if strings.Index(collection["name"].(string), "output") == 0 {
1826                                 manifest := collection["manifest_text"].(string)
1827                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1828 `)
1829                         }
1830                 }
1831         }
1832 }
1833
1834 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1835         helperRecord := `{
1836                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1837                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1838                 "cwd": "/bin",
1839                 "environment": {"FROBIZ": "bilbo"},
1840                 "mounts": {
1841         "/tmp": {"kind": "tmp"},
1842         "stdin": {"kind": "json", "content": "foo"},
1843         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1844     },
1845                 "output_path": "/tmp",
1846                 "priority": 1,
1847                 "runtime_constraints": {},
1848                 "state": "Locked"
1849         }`
1850
1851         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1852                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1853         })
1854
1855         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1856         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1857         for _, v := range api.Content {
1858                 if v["collection"] != nil {
1859                         collection := v["collection"].(arvadosclient.Dict)
1860                         if strings.Index(collection["name"].(string), "output") == 0 {
1861                                 manifest := collection["manifest_text"].(string)
1862                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1863 `)
1864                         }
1865                 }
1866         }
1867 }
1868
1869 func (s *TestSuite) TestStderrMount(c *C) {
1870         api, cr, _ := s.fullRunHelper(c, `{
1871     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1872     "container_image": "`+arvadostest.DockerImage112PDH+`",
1873     "cwd": ".",
1874     "environment": {},
1875     "mounts": {"/tmp": {"kind": "tmp"},
1876                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1877                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1878     "output_path": "/tmp",
1879     "priority": 1,
1880     "runtime_constraints": {},
1881     "state": "Locked"
1882 }`, nil, 1, func() {
1883                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1884                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1885         })
1886
1887         final := api.CalledWith("container.state", "Complete")
1888         c.Assert(final, NotNil)
1889         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1890         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1891
1892         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1893 }
1894
1895 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1896         s.api.callraw = true
1897         err := s.runner.fetchContainerRecord()
1898         c.Assert(err, IsNil)
1899         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1900         c.Logf("%#v", s.runner.Container)
1901         c.Check(err, IsNil)
1902         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1903 }
1904
1905 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1906         nextState := ""
1907         for _, setup := range []func(){
1908                 func() {
1909                         c.Log("// waitErr = ocl runtime error")
1910                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1911                         nextState = "Cancelled"
1912                 },
1913                 func() {
1914                         c.Log("// loadErr = cannot connect")
1915                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1916                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1917                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1918                         c.Assert(err, IsNil)
1919                         nextState = "Queued"
1920                 },
1921         } {
1922                 s.SetUpTest(c)
1923                 setup()
1924                 s.fullRunHelper(c, `{
1925     "command": ["echo", "hello world"],
1926     "container_image": "`+arvadostest.DockerImage112PDH+`",
1927     "cwd": ".",
1928     "environment": {},
1929     "mounts": {"/tmp": {"kind": "tmp"} },
1930     "output_path": "/tmp",
1931     "priority": 1,
1932     "runtime_constraints": {},
1933     "state": "Locked"
1934 }`, nil, 0, func() {})
1935                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1936                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1937                 if *brokenNodeHook != "" {
1938                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1939                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1940                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1941                 } else {
1942                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1943                 }
1944         }
1945 }
1946
1947 func (s *TestSuite) TestBadCommand(c *C) {
1948         for _, startError := range []string{
1949                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1950                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1951                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1952         } {
1953                 s.SetUpTest(c)
1954                 s.executor.startErr = errors.New(startError)
1955                 s.fullRunHelper(c, `{
1956     "command": ["echo", "hello world"],
1957     "container_image": "`+arvadostest.DockerImage112PDH+`",
1958     "cwd": ".",
1959     "environment": {},
1960     "mounts": {"/tmp": {"kind": "tmp"} },
1961     "output_path": "/tmp",
1962     "priority": 1,
1963     "runtime_constraints": {},
1964     "state": "Locked"
1965 }`, nil, 0, func() {})
1966                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1967                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1968         }
1969 }
1970
1971 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1972         helperRecord := `{
1973                 "command": ["true"],
1974                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1975                 "cwd": "/bin",
1976                 "mounts": {
1977                     "/tmp": {"kind": "tmp"},
1978                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1979                 },
1980                 "secret_mounts": {
1981                 },
1982                 "output_path": "/tmp",
1983                 "priority": 1,
1984                 "runtime_constraints": {},
1985                 "state": "Locked"
1986         }`
1987
1988         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1989                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1990                 c.Check(err, IsNil)
1991                 c.Check(string(content), Equals, "mypassword")
1992         })
1993
1994         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1995         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1996         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1997         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1998
1999         // under secret mounts, not captured in output
2000         helperRecord = `{
2001                 "command": ["true"],
2002                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2003                 "cwd": "/bin",
2004                 "mounts": {
2005                     "/tmp": {"kind": "tmp"}
2006                 },
2007                 "secret_mounts": {
2008                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2009                 },
2010                 "output_path": "/tmp",
2011                 "priority": 1,
2012                 "runtime_constraints": {},
2013                 "state": "Locked"
2014         }`
2015
2016         s.SetUpTest(c)
2017         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2018                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2019                 c.Check(err, IsNil)
2020                 c.Check(string(content), Equals, "mypassword")
2021         })
2022
2023         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2024         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2025         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2026         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2027
2028         // under secret mounts, output dir is a collection, not captured in output
2029         helperRecord = `{
2030                 "command": ["true"],
2031                 "container_image": "` + arvadostest.DockerImage112PDH + `",
2032                 "cwd": "/bin",
2033                 "mounts": {
2034                     "/tmp": {"kind": "collection", "writable": true}
2035                 },
2036                 "secret_mounts": {
2037                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
2038                 },
2039                 "output_path": "/tmp",
2040                 "priority": 1,
2041                 "runtime_constraints": {},
2042                 "state": "Locked"
2043         }`
2044
2045         s.SetUpTest(c)
2046         _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
2047                 // secret.conf should be provisioned as a separate
2048                 // bind mount, i.e., it should not appear in the
2049                 // (fake) fuse filesystem as viewed from the host.
2050                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2051                 if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
2052                         c.Logf("secret.conf: content %q, err %#v", content, err)
2053                 }
2054                 err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
2055                 c.Check(err, IsNil)
2056         })
2057
2058         content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
2059         c.Check(err, IsNil)
2060         c.Check(string(content), Equals, "mypassword")
2061         c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
2062         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2063         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2064         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
2065 }
2066
2067 type FakeProcess struct {
2068         cmdLine []string
2069 }
2070
2071 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2072         return fp.cmdLine, nil
2073 }