lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
 116 func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
 117 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 118 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 119 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 120 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 121 func (e *stubExecutor) Close()                          { e.closed = true }
 122 func (e *stubExecutor) Wait(context.Context) (int, error) {
 123         defer e.created.Stdout.Close()
 124         defer e.created.Stderr.Close()
 125         return <-e.exit, e.waitErr
 126 }
 127
 128 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 129
 130 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 131 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 132 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 133
 134 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 135 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 136
 137 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 138 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 139 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 140 `
 141
 142 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 143
 144 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 145 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 146
 147 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 148 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 149
 150 func (client *ArvTestClient) Create(resourceType string,
 151         parameters arvadosclient.Dict,
 152         output interface{}) error {
 153
 154         client.Mutex.Lock()
 155         defer client.Mutex.Unlock()
 156
 157         client.Calls++
 158         client.Content = append(client.Content, parameters)
 159
 160         if resourceType == "logs" {
 161                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 162                 if client.Logs == nil {
 163                         client.Logs = make(map[string]*bytes.Buffer)
 164                 }
 165                 if client.Logs[et] == nil {
 166                         client.Logs[et] = &bytes.Buffer{}
 167                 }
 168                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 169         }
 170
 171         if resourceType == "collections" && output != nil {
 172                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 173                 outmap := output.(*arvados.Collection)
 174                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 175                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 176         }
 177
 178         return nil
 179 }
 180
 181 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 182         switch {
 183         case method == "GET" && resourceType == "containers" && action == "auth":
 184                 return json.Unmarshal([]byte(`{
 185                         "kind": "arvados#api_client_authorization",
 186                         "uuid": "`+fakeAuthUUID+`",
 187                         "api_token": "`+fakeAuthToken+`"
 188                         }`), output)
 189         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 190                 if client.secretMounts != nil {
 191                         return json.Unmarshal(client.secretMounts, output)
 192                 }
 193                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 194         default:
 195                 return fmt.Errorf("Not found")
 196         }
 197 }
 198
 199 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 200         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 201         var j []byte
 202         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 203                 j = []byte(`{
 204                         "kind": "arvados#nodeList",
 205                         "items": [{
 206                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 207                                 "hostname": "compute2",
 208                                 "properties": {"total_cpu_cores": 16}
 209                         }]}`)
 210         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 211                 if uuid == "" {
 212                         j, err = json.Marshal(map[string]interface{}{
 213                                 "items": []interface{}{client.Container},
 214                                 "kind":  "arvados#nodeList",
 215                         })
 216                 } else {
 217                         j, err = json.Marshal(client.Container)
 218                 }
 219         } else {
 220                 j = []byte(`{
 221                         "command": ["sleep", "1"],
 222                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 223                         "cwd": ".",
 224                         "environment": {},
 225                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 226                         "output_path": "/tmp",
 227                         "priority": 1,
 228                         "runtime_constraints": {}
 229                 }`)
 230         }
 231         return ioutil.NopCloser(bytes.NewReader(j)), err
 232 }
 233
 234 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 235         if resourceType == "collections" {
 236                 if uuid == hwPDH {
 237                         output.(*arvados.Collection).ManifestText = hwManifest
 238                 } else if uuid == otherPDH {
 239                         output.(*arvados.Collection).ManifestText = otherManifest
 240                 } else if uuid == normalizedWithSubdirsPDH {
 241                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 242                 } else if uuid == denormalizedWithSubdirsPDH {
 243                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 244                 }
 245         }
 246         if resourceType == "containers" {
 247                 (*output.(*arvados.Container)) = client.Container
 248         }
 249         return nil
 250 }
 251
 252 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 253         client.Mutex.Lock()
 254         defer client.Mutex.Unlock()
 255         client.Calls++
 256         client.Content = append(client.Content, parameters)
 257         if resourceType == "containers" {
 258                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 259                         client.WasSetRunning = true
 260                 }
 261         } else if resourceType == "collections" {
 262                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 263                 output.(*arvados.Collection).UUID = uuid
 264                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 265         }
 266         return nil
 267 }
 268
 269 var discoveryMap = map[string]interface{}{
 270         "defaultTrashLifetime":               float64(1209600),
 271         "crunchLimitLogBytesPerJob":          float64(67108864),
 272         "crunchLogThrottleBytes":             float64(65536),
 273         "crunchLogThrottlePeriod":            float64(60),
 274         "crunchLogThrottleLines":             float64(1024),
 275         "crunchLogPartialLineThrottlePeriod": float64(5),
 276         "crunchLogBytesPerEvent":             float64(4096),
 277         "crunchLogSecondsBetweenEvents":      float64(1),
 278 }
 279
 280 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 281         return discoveryMap[key], nil
 282 }
 283
 284 // CalledWith returns the parameters from the first API call whose
 285 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 286 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 287 // no call matches, it returns nil.
 288 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 289 call:
 290         for _, content := range client.Content {
 291                 var v interface{} = content
 292                 for _, k := range strings.Split(jpath, ".") {
 293                         if dict, ok := v.(arvadosclient.Dict); !ok {
 294                                 continue call
 295                         } else {
 296                                 v = dict[k]
 297                         }
 298                 }
 299                 if v == expect {
 300                         return content
 301                 }
 302         }
 303         return nil
 304 }
 305
 306 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 307         return locator, nil
 308 }
 309
 310 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 311         client.Content = opts.Data
 312         return arvados.BlockWriteResponse{
 313                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 314         }, nil
 315 }
 316
 317 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 318         return 0, errors.New("not implemented")
 319 }
 320
 321 func (client *KeepTestClient) ClearBlockCache() {
 322 }
 323
 324 func (client *KeepTestClient) Close() {
 325         client.Content = nil
 326 }
 327
 328 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 329         client.StorageClasses = sc
 330 }
 331
 332 type FileWrapper struct {
 333         io.ReadCloser
 334         len int64
 335 }
 336
 337 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 338         return nil, errors.New("not implemented")
 339 }
 340
 341 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 342         return 0, errors.New("not implemented")
 343 }
 344
 345 func (fw FileWrapper) Size() int64 {
 346         return fw.len
 347 }
 348
 349 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 350         return nil, errors.New("not implemented")
 351 }
 352
 353 func (fw FileWrapper) Truncate(int64) error {
 354         return errors.New("not implemented")
 355 }
 356
 357 func (fw FileWrapper) Write([]byte) (int, error) {
 358         return 0, errors.New("not implemented")
 359 }
 360
 361 func (fw FileWrapper) Sync() error {
 362         return errors.New("not implemented")
 363 }
 364
 365 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 366         if filename == hwImageID+".tar" {
 367                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 368                 client.Called = true
 369                 return FileWrapper{rdr, 1321984}, nil
 370         } else if filename == "/file1_in_main.txt" {
 371                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 372                 client.Called = true
 373                 return FileWrapper{rdr, 3}, nil
 374         }
 375         return nil, nil
 376 }
 377
 378 func (s *TestSuite) TestLoadImage(c *C) {
 379         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 380         s.runner.Container.Mounts = map[string]arvados.Mount{
 381                 "/out": {Kind: "tmp", Writable: true},
 382         }
 383         s.runner.Container.OutputPath = "/out"
 384
 385         _, err := s.runner.SetupMounts()
 386         c.Assert(err, IsNil)
 387
 388         imageID, err := s.runner.LoadImage()
 389         c.Check(err, IsNil)
 390         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 391         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 392
 393         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 394         s.executor.imageLoaded = false
 395         s.executor.loaded = ""
 396         s.executor.loadErr = errors.New("bork")
 397         imageID, err = s.runner.LoadImage()
 398         c.Check(err, ErrorMatches, ".*bork")
 399         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 400
 401         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 402         s.executor.imageLoaded = false
 403         s.executor.loaded = ""
 404         s.executor.loadErr = nil
 405         imageID, err = s.runner.LoadImage()
 406         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 407         c.Check(s.executor.loaded, Equals, "")
 408
 409         // if executor reports image is already loaded, LoadImage should not be called
 410         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 411         s.executor.imageLoaded = true
 412         s.executor.loaded = ""
 413         s.executor.loadErr = nil
 414         imageID, err = s.runner.LoadImage()
 415         c.Check(err, IsNil)
 416         c.Check(s.executor.loaded, Equals, "")
 417         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 418 }
 419
 420 type ArvErrorTestClient struct{}
 421
 422 func (ArvErrorTestClient) Create(resourceType string,
 423         parameters arvadosclient.Dict,
 424         output interface{}) error {
 425         return nil
 426 }
 427
 428 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 429         if method == "GET" && resourceType == "containers" && action == "auth" {
 430                 return nil
 431         }
 432         return errors.New("ArvError")
 433 }
 434
 435 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 436         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 437         return nil, errors.New("ArvError")
 438 }
 439
 440 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 441         return errors.New("ArvError")
 442 }
 443
 444 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 445         return nil
 446 }
 447
 448 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 449         return discoveryMap[key], nil
 450 }
 451
 452 type KeepErrorTestClient struct {
 453         KeepTestClient
 454 }
 455
 456 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 457         return nil, errors.New("KeepError")
 458 }
 459
 460 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 461         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 462 }
 463
 464 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 465         return "", errors.New("KeepError")
 466 }
 467
 468 type KeepReadErrorTestClient struct {
 469         KeepTestClient
 470 }
 471
 472 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 473         return 0, errors.New("KeepError")
 474 }
 475
 476 type ErrorReader struct {
 477         FileWrapper
 478 }
 479
 480 func (ErrorReader) Read(p []byte) (n int, err error) {
 481         return 0, errors.New("ErrorReader")
 482 }
 483
 484 func (ErrorReader) Seek(int64, int) (int64, error) {
 485         return 0, errors.New("ErrorReader")
 486 }
 487
 488 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 489         return ErrorReader{}, nil
 490 }
 491
 492 type ClosableBuffer struct {
 493         bytes.Buffer
 494 }
 495
 496 func (*ClosableBuffer) Close() error {
 497         return nil
 498 }
 499
 500 type TestLogs struct {
 501         Stdout ClosableBuffer
 502         Stderr ClosableBuffer
 503 }
 504
 505 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 506         if logstr == "stdout" {
 507                 return &tl.Stdout, nil
 508         }
 509         if logstr == "stderr" {
 510                 return &tl.Stderr, nil
 511         }
 512         return nil, errors.New("???")
 513 }
 514
 515 func dockerLog(fd byte, msg string) []byte {
 516         by := []byte(msg)
 517         header := make([]byte, 8+len(by))
 518         header[0] = fd
 519         header[7] = byte(len(by))
 520         copy(header[8:], by)
 521         return header
 522 }
 523
 524 func (s *TestSuite) TestRunContainer(c *C) {
 525         s.executor.runFunc = func() {
 526                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 527                 s.executor.created.Stdout.Close()
 528                 s.executor.created.Stderr.Close()
 529                 s.executor.exit <- 0
 530         }
 531
 532         var logs TestLogs
 533         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 534         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 535         s.runner.Container.Command = []string{"./hw"}
 536         s.runner.Container.OutputStorageClasses = []string{"default"}
 537
 538         imageID, err := s.runner.LoadImage()
 539         c.Assert(err, IsNil)
 540
 541         err = s.runner.CreateContainer(imageID, nil)
 542         c.Assert(err, IsNil)
 543
 544         err = s.runner.StartContainer()
 545         c.Assert(err, IsNil)
 546
 547         err = s.runner.WaitFinish()
 548         c.Assert(err, IsNil)
 549
 550         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 551         c.Check(logs.Stderr.String(), Equals, "")
 552 }
 553
 554 func (s *TestSuite) TestCommitLogs(c *C) {
 555         api := &ArvTestClient{}
 556         kc := &KeepTestClient{}
 557         defer kc.Close()
 558         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 559         c.Assert(err, IsNil)
 560         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 561
 562         cr.CrunchLog.Print("Hello world!")
 563         cr.CrunchLog.Print("Goodbye")
 564         cr.finalState = "Complete"
 565
 566         err = cr.CommitLogs()
 567         c.Check(err, IsNil)
 568
 569         c.Check(api.Calls, Equals, 2)
 570         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 571         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 572         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 573         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 574 }
 575
 576 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 577         api := &ArvTestClient{}
 578         kc := &KeepTestClient{}
 579         defer kc.Close()
 580         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 581         c.Assert(err, IsNil)
 582
 583         err = cr.UpdateContainerRunning()
 584         c.Check(err, IsNil)
 585
 586         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 587 }
 588
 589 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 590         api := &ArvTestClient{}
 591         kc := &KeepTestClient{}
 592         defer kc.Close()
 593         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 594         c.Assert(err, IsNil)
 595
 596         cr.LogsPDH = new(string)
 597         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 598
 599         cr.ExitCode = new(int)
 600         *cr.ExitCode = 42
 601         cr.finalState = "Complete"
 602
 603         err = cr.UpdateContainerFinal()
 604         c.Check(err, IsNil)
 605
 606         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 607         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 608         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 609 }
 610
 611 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 612         api := &ArvTestClient{}
 613         kc := &KeepTestClient{}
 614         defer kc.Close()
 615         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 616         c.Assert(err, IsNil)
 617         cr.cCancelled = true
 618         cr.finalState = "Cancelled"
 619
 620         err = cr.UpdateContainerFinal()
 621         c.Check(err, IsNil)
 622
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 624         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 625         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 626 }
 627
 628 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 629 // dress rehearsal of the Run() function, starting from a JSON container record.
 630 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 631         err := json.Unmarshal([]byte(record), &s.api.Container)
 632         c.Assert(err, IsNil)
 633         initialState := s.api.Container.State
 634
 635         var sm struct {
 636                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 637         }
 638         err = json.Unmarshal([]byte(record), &sm)
 639         c.Check(err, IsNil)
 640         secretMounts, err := json.Marshal(sm)
 641         c.Assert(err, IsNil)
 642         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 643
 644         s.executor.runFunc = func() {
 645                 fn()
 646                 s.executor.exit <- exitCode
 647         }
 648
 649         s.runner.statInterval = 100 * time.Millisecond
 650         s.runner.containerWatchdogInterval = time.Second
 651         am := &ArvMountCmdLine{}
 652         s.runner.RunArvMount = am.ArvMountTest
 653
 654         realTemp := c.MkDir()
 655         tempcount := 0
 656         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 657                 tempcount++
 658                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 659                 err := os.Mkdir(d, os.ModePerm)
 660                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 661                         // Test case must have pre-populated the tempdir
 662                         err = nil
 663                 }
 664                 return d, err
 665         }
 666         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 667                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 668         }
 669
 670         if extraMounts != nil && len(extraMounts) > 0 {
 671                 err := s.runner.SetupArvMountPoint("keep")
 672                 c.Check(err, IsNil)
 673
 674                 for _, m := range extraMounts {
 675                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 676                 }
 677         }
 678
 679         err = s.runner.Run()
 680         if s.api.CalledWith("container.state", "Complete") != nil {
 681                 c.Check(err, IsNil)
 682         }
 683         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 684                 c.Check(s.api.WasSetRunning, Equals, true)
 685                 var lastupdate arvadosclient.Dict
 686                 for _, content := range s.api.Content {
 687                         if content["container"] != nil {
 688                                 lastupdate = content["container"].(arvadosclient.Dict)
 689                         }
 690                 }
 691                 if lastupdate["log"] == nil {
 692                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 693                 }
 694         }
 695
 696         if err != nil {
 697                 for k, v := range s.api.Logs {
 698                         c.Log(k)
 699                         c.Log(v.String())
 700                 }
 701         }
 702
 703         return s.api, s.runner, realTemp
 704 }
 705
 706 func (s *TestSuite) TestFullRunHello(c *C) {
 707         s.runner.enableMemoryLimit = true
 708         s.runner.networkMode = "default"
 709         s.fullRunHelper(c, `{
 710     "command": ["echo", "hello world"],
 711     "container_image": "`+arvadostest.DockerImage112PDH+`",
 712     "cwd": ".",
 713     "environment": {"foo":"bar","baz":"waz"},
 714     "mounts": {"/tmp": {"kind": "tmp"} },
 715     "output_path": "/tmp",
 716     "priority": 1,
 717     "runtime_constraints": {"vcpus":1,"ram":1000000},
 718     "state": "Locked",
 719     "output_storage_classes": ["default"]
 720 }`, nil, 0, func() {
 721                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 722                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 723                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 724                 c.Check(s.executor.created.VCPUs, Equals, 1)
 725                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 726                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 727                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 728                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 729         })
 730
 731         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 732         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 733         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 734         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 735         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 736 }
 737
 738 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 739         var ran bool
 740         s.fullRunHelper(c, `{
 741     "command": ["sleep", "3"],
 742     "container_image": "`+arvadostest.DockerImage112PDH+`",
 743     "cwd": ".",
 744     "environment": {},
 745     "mounts": {"/tmp": {"kind": "tmp"} },
 746     "output_path": "/tmp",
 747     "priority": 1,
 748     "runtime_constraints": {},
 749     "scheduling_parameters":{"max_run_time": 1},
 750     "state": "Running"
 751 }`, nil, 2, func() {
 752                 ran = true
 753         })
 754         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 755         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 756         c.Check(ran, Equals, false)
 757 }
 758
 759 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 760         s.fullRunHelper(c, `{
 761     "command": ["sleep", "3"],
 762     "container_image": "`+arvadostest.DockerImage112PDH+`",
 763     "cwd": ".",
 764     "environment": {},
 765     "mounts": {"/tmp": {"kind": "tmp"} },
 766     "output_path": "/tmp",
 767     "priority": 1,
 768     "runtime_constraints": {},
 769     "scheduling_parameters":{"max_run_time": 1},
 770     "state": "Locked"
 771 }`, nil, 0, func() {
 772                 time.Sleep(3 * time.Second)
 773         })
 774
 775         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 776         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 777 }
 778
 779 func (s *TestSuite) TestContainerWaitFails(c *C) {
 780         s.fullRunHelper(c, `{
 781     "command": ["sleep", "3"],
 782     "container_image": "`+arvadostest.DockerImage112PDH+`",
 783     "cwd": ".",
 784     "mounts": {"/tmp": {"kind": "tmp"} },
 785     "output_path": "/tmp",
 786     "priority": 1,
 787     "state": "Locked"
 788 }`, nil, 0, func() {
 789                 s.executor.waitErr = errors.New("Container is not running")
 790         })
 791
 792         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 793         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 794 }
 795
 796 func (s *TestSuite) TestCrunchstat(c *C) {
 797         s.fullRunHelper(c, `{
 798                 "command": ["sleep", "1"],
 799                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 800                 "cwd": ".",
 801                 "environment": {},
 802                 "mounts": {"/tmp": {"kind": "tmp"} },
 803                 "output_path": "/tmp",
 804                 "priority": 1,
 805                 "runtime_constraints": {},
 806                 "state": "Locked"
 807         }`, nil, 0, func() {
 808                 time.Sleep(time.Second)
 809         })
 810
 811         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 812         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 813
 814         // We didn't actually start a container, so crunchstat didn't
 815         // find accounting files and therefore didn't log any stats.
 816         // It should have logged a "can't find accounting files"
 817         // message after one poll interval, though, so we can confirm
 818         // it's alive:
 819         c.Assert(s.api.Logs["crunchstat"], NotNil)
 820         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 821
 822         // The "files never appeared" log assures us that we called
 823         // (*crunchstat.Reporter)Stop(), and that we set it up with
 824         // the correct container ID "abcde":
 825         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 826 }
 827
 828 func (s *TestSuite) TestNodeInfoLog(c *C) {
 829         os.Setenv("SLURMD_NODENAME", "compute2")
 830         s.fullRunHelper(c, `{
 831                 "command": ["sleep", "1"],
 832                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 833                 "cwd": ".",
 834                 "environment": {},
 835                 "mounts": {"/tmp": {"kind": "tmp"} },
 836                 "output_path": "/tmp",
 837                 "priority": 1,
 838                 "runtime_constraints": {},
 839                 "state": "Locked"
 840         }`, nil, 0,
 841                 func() {
 842                         time.Sleep(time.Second)
 843                 })
 844
 845         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 846         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 847
 848         c.Assert(s.api.Logs["node"], NotNil)
 849         json := s.api.Logs["node"].String()
 850         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 851         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 852         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 853
 854         c.Assert(s.api.Logs["node-info"], NotNil)
 855         json = s.api.Logs["node-info"].String()
 856         c.Check(json, Matches, `(?ms).*Host Information.*`)
 857         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 858         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 859         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 860         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 861 }
 862
 863 func (s *TestSuite) TestContainerRecordLog(c *C) {
 864         s.fullRunHelper(c, `{
 865                 "command": ["sleep", "1"],
 866                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 867                 "cwd": ".",
 868                 "environment": {},
 869                 "mounts": {"/tmp": {"kind": "tmp"} },
 870                 "output_path": "/tmp",
 871                 "priority": 1,
 872                 "runtime_constraints": {},
 873                 "state": "Locked"
 874         }`, nil, 0,
 875                 func() {
 876                         time.Sleep(time.Second)
 877                 })
 878
 879         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 880         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 881
 882         c.Assert(s.api.Logs["container"], NotNil)
 883         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 884 }
 885
 886 func (s *TestSuite) TestFullRunStderr(c *C) {
 887         s.fullRunHelper(c, `{
 888     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 889     "container_image": "`+arvadostest.DockerImage112PDH+`",
 890     "cwd": ".",
 891     "environment": {},
 892     "mounts": {"/tmp": {"kind": "tmp"} },
 893     "output_path": "/tmp",
 894     "priority": 1,
 895     "runtime_constraints": {},
 896     "state": "Locked"
 897 }`, nil, 1, func() {
 898                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 899                 fmt.Fprintln(s.executor.created.Stderr, "world")
 900         })
 901
 902         final := s.api.CalledWith("container.state", "Complete")
 903         c.Assert(final, NotNil)
 904         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 905         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 906
 907         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 908         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 909 }
 910
 911 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 912         s.fullRunHelper(c, `{
 913     "command": ["pwd"],
 914     "container_image": "`+arvadostest.DockerImage112PDH+`",
 915     "cwd": ".",
 916     "environment": {},
 917     "mounts": {"/tmp": {"kind": "tmp"} },
 918     "output_path": "/tmp",
 919     "priority": 1,
 920     "runtime_constraints": {},
 921     "state": "Locked"
 922 }`, nil, 0, func() {
 923                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 924         })
 925
 926         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 927         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 928         c.Log(s.api.Logs["stdout"])
 929         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 930 }
 931
 932 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 933         s.fullRunHelper(c, `{
 934     "command": ["pwd"],
 935     "container_image": "`+arvadostest.DockerImage112PDH+`",
 936     "cwd": "/bin",
 937     "environment": {},
 938     "mounts": {"/tmp": {"kind": "tmp"} },
 939     "output_path": "/tmp",
 940     "priority": 1,
 941     "runtime_constraints": {},
 942     "state": "Locked"
 943 }`, nil, 0, func() {
 944                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 945         })
 946
 947         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 948         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 949         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 950 }
 951
 952 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 953         s.fullRunHelper(c, `{
 954     "command": ["pwd"],
 955     "container_image": "`+arvadostest.DockerImage112PDH+`",
 956     "cwd": "/bin",
 957     "environment": {},
 958     "mounts": {"/tmp": {"kind": "tmp"} },
 959     "output_path": "/tmp",
 960     "priority": 1,
 961     "runtime_constraints": {},
 962     "state": "Locked",
 963     "output_storage_classes": ["foo", "bar"]
 964 }`, nil, 0, func() {
 965                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 966         })
 967
 968         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 969         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 970         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 971         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 972         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 973 }
 974
 975 func (s *TestSuite) TestStopOnSignal(c *C) {
 976         s.executor.runFunc = func() {
 977                 s.executor.created.Stdout.Write([]byte("foo\n"))
 978                 s.runner.SigChan <- syscall.SIGINT
 979         }
 980         s.testStopContainer(c)
 981 }
 982
 983 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 984         s.executor.runFunc = func() {
 985                 s.executor.created.Stdout.Write([]byte("foo\n"))
 986                 s.runner.ArvMountExit <- nil
 987                 close(s.runner.ArvMountExit)
 988         }
 989         s.runner.ArvMountExit = make(chan error)
 990         s.testStopContainer(c)
 991 }
 992
 993 func (s *TestSuite) testStopContainer(c *C) {
 994         record := `{
 995     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 996     "container_image": "` + arvadostest.DockerImage112PDH + `",
 997     "cwd": ".",
 998     "environment": {},
 999     "mounts": {"/tmp": {"kind": "tmp"} },
1000     "output_path": "/tmp",
1001     "priority": 1,
1002     "runtime_constraints": {},
1003     "state": "Locked"
1004 }`
1005
1006         err := json.Unmarshal([]byte(record), &s.api.Container)
1007         c.Assert(err, IsNil)
1008
1009         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1010         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1011                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1012         }
1013
1014         done := make(chan error)
1015         go func() {
1016                 done <- s.runner.Run()
1017         }()
1018         select {
1019         case <-time.After(20 * time.Second):
1020                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1021                 c.Fatal("timed out")
1022         case err = <-done:
1023                 c.Check(err, IsNil)
1024         }
1025         for k, v := range s.api.Logs {
1026                 c.Log(k)
1027                 c.Log(v.String(), "\n")
1028         }
1029
1030         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1031         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1032         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1033 }
1034
1035 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1036         s.fullRunHelper(c, `{
1037     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1038     "container_image": "`+arvadostest.DockerImage112PDH+`",
1039     "cwd": "/bin",
1040     "environment": {"FROBIZ": "bilbo"},
1041     "mounts": {"/tmp": {"kind": "tmp"} },
1042     "output_path": "/tmp",
1043     "priority": 1,
1044     "runtime_constraints": {},
1045     "state": "Locked"
1046 }`, nil, 0, func() {
1047                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1048         })
1049
1050         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1051         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1052         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1053 }
1054
1055 type ArvMountCmdLine struct {
1056         Cmd   []string
1057         token string
1058 }
1059
1060 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1061         am.Cmd = c
1062         am.token = token
1063         return nil, nil
1064 }
1065
1066 func stubCert(temp string) string {
1067         path := temp + "/ca-certificates.crt"
1068         crt, _ := os.Create(path)
1069         crt.Close()
1070         arvadosclient.CertFiles = []string{path}
1071         return path
1072 }
1073
1074 func (s *TestSuite) TestSetupMounts(c *C) {
1075         cr := s.runner
1076         am := &ArvMountCmdLine{}
1077         cr.RunArvMount = am.ArvMountTest
1078         cr.ContainerArvClient = &ArvTestClient{}
1079         cr.ContainerKeepClient = &KeepTestClient{}
1080         cr.Container.OutputStorageClasses = []string{"default"}
1081
1082         realTemp := c.MkDir()
1083         certTemp := c.MkDir()
1084         stubCertPath := stubCert(certTemp)
1085         cr.parentTemp = realTemp
1086
1087         i := 0
1088         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1089                 i++
1090                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1091                 err := os.Mkdir(d, os.ModePerm)
1092                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1093                         // Test case must have pre-populated the tempdir
1094                         err = nil
1095                 }
1096                 return d, err
1097         }
1098
1099         checkEmpty := func() {
1100                 // Should be deleted.
1101                 _, err := os.Stat(realTemp)
1102                 c.Assert(os.IsNotExist(err), Equals, true)
1103
1104                 // Now recreate it for the next test.
1105                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1106         }
1107
1108         {
1109                 i = 0
1110                 cr.ArvMountPoint = ""
1111                 cr.Container.Mounts = make(map[string]arvados.Mount)
1112                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1113                 cr.Container.OutputPath = "/tmp"
1114                 cr.statInterval = 5 * time.Second
1115                 bindmounts, err := cr.SetupMounts()
1116                 c.Check(err, IsNil)
1117                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1118                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1119                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1120                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1121                 os.RemoveAll(cr.ArvMountPoint)
1122                 cr.CleanupDirs()
1123                 checkEmpty()
1124         }
1125
1126         {
1127                 i = 0
1128                 cr.ArvMountPoint = ""
1129                 cr.Container.Mounts = make(map[string]arvados.Mount)
1130                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1131                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1132                 cr.Container.OutputPath = "/out"
1133                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1134
1135                 bindmounts, err := cr.SetupMounts()
1136                 c.Check(err, IsNil)
1137                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1138                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1139                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1140                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1141                 os.RemoveAll(cr.ArvMountPoint)
1142                 cr.CleanupDirs()
1143                 checkEmpty()
1144         }
1145
1146         {
1147                 i = 0
1148                 cr.ArvMountPoint = ""
1149                 cr.Container.Mounts = make(map[string]arvados.Mount)
1150                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1151                 cr.Container.OutputPath = "/tmp"
1152                 cr.Container.RuntimeConstraints.API = true
1153                 cr.Container.OutputStorageClasses = []string{"default"}
1154
1155                 bindmounts, err := cr.SetupMounts()
1156                 c.Check(err, IsNil)
1157                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1158                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1159                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1160                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1161                 os.RemoveAll(cr.ArvMountPoint)
1162                 cr.CleanupDirs()
1163                 checkEmpty()
1164
1165                 cr.Container.RuntimeConstraints.API = false
1166         }
1167
1168         {
1169                 i = 0
1170                 cr.ArvMountPoint = ""
1171                 cr.Container.Mounts = map[string]arvados.Mount{
1172                         "/keeptmp": {Kind: "collection", Writable: true},
1173                 }
1174                 cr.Container.OutputPath = "/keeptmp"
1175
1176                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1177
1178                 bindmounts, err := cr.SetupMounts()
1179                 c.Check(err, IsNil)
1180                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1181                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1182                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1183                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1184                 os.RemoveAll(cr.ArvMountPoint)
1185                 cr.CleanupDirs()
1186                 checkEmpty()
1187         }
1188
1189         {
1190                 i = 0
1191                 cr.ArvMountPoint = ""
1192                 cr.Container.Mounts = map[string]arvados.Mount{
1193                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1194                         "/keepout": {Kind: "collection", Writable: true},
1195                 }
1196                 cr.Container.OutputPath = "/keepout"
1197
1198                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1199                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1200
1201                 bindmounts, err := cr.SetupMounts()
1202                 c.Check(err, IsNil)
1203                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1204                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1205                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1206                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1207                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1208                         "/keepout": {realTemp + "/keep1/tmp0", false},
1209                 })
1210                 os.RemoveAll(cr.ArvMountPoint)
1211                 cr.CleanupDirs()
1212                 checkEmpty()
1213         }
1214
1215         {
1216                 i = 0
1217                 cr.ArvMountPoint = ""
1218                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1219                 cr.Container.Mounts = map[string]arvados.Mount{
1220                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1221                         "/keepout": {Kind: "collection", Writable: true},
1222                 }
1223                 cr.Container.OutputPath = "/keepout"
1224
1225                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1226                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1227
1228                 bindmounts, err := cr.SetupMounts()
1229                 c.Check(err, IsNil)
1230                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1231                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1232                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1233                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1234                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1235                         "/keepout": {realTemp + "/keep1/tmp0", false},
1236                 })
1237                 os.RemoveAll(cr.ArvMountPoint)
1238                 cr.CleanupDirs()
1239                 checkEmpty()
1240         }
1241
1242         for _, test := range []struct {
1243                 in  interface{}
1244                 out string
1245         }{
1246                 {in: "foo", out: `"foo"`},
1247                 {in: nil, out: `null`},
1248                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1249         } {
1250                 i = 0
1251                 cr.ArvMountPoint = ""
1252                 cr.Container.Mounts = map[string]arvados.Mount{
1253                         "/mnt/test.json": {Kind: "json", Content: test.in},
1254                 }
1255                 bindmounts, err := cr.SetupMounts()
1256                 c.Check(err, IsNil)
1257                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1258                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1259                 })
1260                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1261                 c.Check(err, IsNil)
1262                 c.Check(content, DeepEquals, []byte(test.out))
1263                 os.RemoveAll(cr.ArvMountPoint)
1264                 cr.CleanupDirs()
1265                 checkEmpty()
1266         }
1267
1268         for _, test := range []struct {
1269                 in  interface{}
1270                 out string
1271         }{
1272                 {in: "foo", out: `foo`},
1273                 {in: nil, out: "error"},
1274                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1275         } {
1276                 i = 0
1277                 cr.ArvMountPoint = ""
1278                 cr.Container.Mounts = map[string]arvados.Mount{
1279                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1280                 }
1281                 bindmounts, err := cr.SetupMounts()
1282                 if test.out == "error" {
1283                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1284                 } else {
1285                         c.Check(err, IsNil)
1286                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1287                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1288                         })
1289                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1290                         c.Check(err, IsNil)
1291                         c.Check(content, DeepEquals, []byte(test.out))
1292                 }
1293                 os.RemoveAll(cr.ArvMountPoint)
1294                 cr.CleanupDirs()
1295                 checkEmpty()
1296         }
1297
1298         // Read-only mount points are allowed underneath output_dir mount point
1299         {
1300                 i = 0
1301                 cr.ArvMountPoint = ""
1302                 cr.Container.Mounts = make(map[string]arvados.Mount)
1303                 cr.Container.Mounts = map[string]arvados.Mount{
1304                         "/tmp":     {Kind: "tmp"},
1305                         "/tmp/foo": {Kind: "collection"},
1306                 }
1307                 cr.Container.OutputPath = "/tmp"
1308
1309                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1310
1311                 bindmounts, err := cr.SetupMounts()
1312                 c.Check(err, IsNil)
1313                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1314                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1315                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1316                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1317                         "/tmp":     {realTemp + "/tmp2", false},
1318                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1319                 })
1320                 os.RemoveAll(cr.ArvMountPoint)
1321                 cr.CleanupDirs()
1322                 checkEmpty()
1323         }
1324
1325         // Writable mount points copied to output_dir mount point
1326         {
1327                 i = 0
1328                 cr.ArvMountPoint = ""
1329                 cr.Container.Mounts = make(map[string]arvados.Mount)
1330                 cr.Container.Mounts = map[string]arvados.Mount{
1331                         "/tmp": {Kind: "tmp"},
1332                         "/tmp/foo": {Kind: "collection",
1333                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1334                                 Writable:         true},
1335                         "/tmp/bar": {Kind: "collection",
1336                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1337                                 Path:             "baz",
1338                                 Writable:         true},
1339                 }
1340                 cr.Container.OutputPath = "/tmp"
1341
1342                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1343                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1344
1345                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1346                 rf.Write([]byte("bar"))
1347                 rf.Close()
1348
1349                 _, err := cr.SetupMounts()
1350                 c.Check(err, IsNil)
1351                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1352                 c.Check(err, IsNil)
1353                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1354                 c.Check(err, IsNil)
1355                 os.RemoveAll(cr.ArvMountPoint)
1356                 cr.CleanupDirs()
1357                 checkEmpty()
1358         }
1359
1360         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1361         {
1362                 i = 0
1363                 cr.ArvMountPoint = ""
1364                 cr.Container.Mounts = make(map[string]arvados.Mount)
1365                 cr.Container.Mounts = map[string]arvados.Mount{
1366                         "/tmp":     {Kind: "tmp"},
1367                         "/tmp/foo": {Kind: "tmp"},
1368                 }
1369                 cr.Container.OutputPath = "/tmp"
1370
1371                 _, err := cr.SetupMounts()
1372                 c.Check(err, NotNil)
1373                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1374                 os.RemoveAll(cr.ArvMountPoint)
1375                 cr.CleanupDirs()
1376                 checkEmpty()
1377         }
1378
1379         // Only mount point of kind 'collection' is allowed for stdin
1380         {
1381                 i = 0
1382                 cr.ArvMountPoint = ""
1383                 cr.Container.Mounts = make(map[string]arvados.Mount)
1384                 cr.Container.Mounts = map[string]arvados.Mount{
1385                         "stdin": {Kind: "tmp"},
1386                 }
1387
1388                 _, err := cr.SetupMounts()
1389                 c.Check(err, NotNil)
1390                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1391                 os.RemoveAll(cr.ArvMountPoint)
1392                 cr.CleanupDirs()
1393                 checkEmpty()
1394         }
1395
1396         // git_tree mounts
1397         {
1398                 i = 0
1399                 cr.ArvMountPoint = ""
1400                 (*GitMountSuite)(nil).useTestGitServer(c)
1401                 cr.token = arvadostest.ActiveToken
1402                 cr.Container.Mounts = make(map[string]arvados.Mount)
1403                 cr.Container.Mounts = map[string]arvados.Mount{
1404                         "/tip": {
1405                                 Kind:   "git_tree",
1406                                 UUID:   arvadostest.Repository2UUID,
1407                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1408                                 Path:   "/",
1409                         },
1410                         "/non-tip": {
1411                                 Kind:   "git_tree",
1412                                 UUID:   arvadostest.Repository2UUID,
1413                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1414                                 Path:   "/",
1415                         },
1416                 }
1417                 cr.Container.OutputPath = "/tmp"
1418
1419                 bindmounts, err := cr.SetupMounts()
1420                 c.Check(err, IsNil)
1421
1422                 for path, mount := range bindmounts {
1423                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1424                 }
1425
1426                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1427                 c.Check(err, IsNil)
1428                 c.Check(string(data), Equals, "\000\001\002\003")
1429                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1430                 c.Check(err, FitsTypeOf, &os.PathError{})
1431                 c.Check(os.IsNotExist(err), Equals, true)
1432
1433                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1434                 c.Check(err, IsNil)
1435                 c.Check(string(data), Equals, "\000\001\002\003")
1436                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1437                 c.Check(err, IsNil)
1438                 c.Check(string(data), Equals, "testfile\n")
1439
1440                 cr.CleanupDirs()
1441                 checkEmpty()
1442         }
1443 }
1444
1445 func (s *TestSuite) TestStdout(c *C) {
1446         helperRecord := `{
1447                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1448                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1449                 "cwd": "/bin",
1450                 "environment": {"FROBIZ": "bilbo"},
1451                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1452                 "output_path": "/tmp",
1453                 "priority": 1,
1454                 "runtime_constraints": {},
1455                 "state": "Locked"
1456         }`
1457
1458         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1459                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1460         })
1461
1462         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1463         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1464         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1465 }
1466
1467 // Used by the TestStdoutWithWrongPath*()
1468 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1469         err := json.Unmarshal([]byte(record), &s.api.Container)
1470         c.Assert(err, IsNil)
1471         s.executor.runFunc = fn
1472         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1473         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1474                 return s.api, &KeepTestClient{}, nil, nil
1475         }
1476         return s.api, s.runner, s.runner.Run()
1477 }
1478
1479 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1480         _, _, err := s.stdoutErrorRunHelper(c, `{
1481     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1482     "output_path": "/tmp",
1483     "state": "Locked"
1484 }`, func() {})
1485         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1486 }
1487
1488 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1489         _, _, err := s.stdoutErrorRunHelper(c, `{
1490     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1491     "output_path": "/tmp",
1492     "state": "Locked"
1493 }`, func() {})
1494         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1495 }
1496
1497 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1498         _, _, err := s.stdoutErrorRunHelper(c, `{
1499     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1500     "output_path": "/tmp",
1501     "state": "Locked"
1502 }`, func() {})
1503         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1504 }
1505
1506 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1507         s.fullRunHelper(c, `{
1508     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1509     "container_image": "`+arvadostest.DockerImage112PDH+`",
1510     "cwd": "/bin",
1511     "environment": {},
1512     "mounts": {"/tmp": {"kind": "tmp"} },
1513     "output_path": "/tmp",
1514     "priority": 1,
1515     "runtime_constraints": {"API": true},
1516     "state": "Locked"
1517 }`, nil, 0, func() {
1518                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1519                 s.executor.exit <- 3
1520         })
1521         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1522         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1523 }
1524
1525 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1526         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1527         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1528         s.fullRunHelper(c, `{
1529     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1530     "container_image": "`+arvadostest.DockerImage112PDH+`",
1531     "cwd": "/bin",
1532     "environment": {},
1533     "mounts": {"/tmp": {"kind": "tmp"} },
1534     "output_path": "/tmp",
1535     "priority": 1,
1536     "runtime_constraints": {"API": true},
1537     "state": "Locked"
1538 }`, nil, 0, func() {
1539                 s.api.Container.Output = arvadostest.DockerImage112PDH
1540         })
1541
1542         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1543         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1544         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1545 }
1546
1547 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1548         helperRecord := `{
1549                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1550                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1551                 "cwd": "/bin",
1552                 "environment": {"FROBIZ": "bilbo"},
1553                 "mounts": {
1554         "/tmp": {"kind": "tmp"},
1555         "/tmp/foo": {"kind": "collection",
1556                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1557                      "exclude_from_output": true
1558         },
1559         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1560     },
1561                 "output_path": "/tmp",
1562                 "priority": 1,
1563                 "runtime_constraints": {},
1564                 "state": "Locked"
1565         }`
1566
1567         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1568
1569         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1570                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1571         })
1572
1573         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1574         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1575         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1576 }
1577
1578 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1579         helperRecord := `{
1580                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1581                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1582                 "cwd": "/bin",
1583                 "environment": {"FROBIZ": "bilbo"},
1584                 "mounts": {
1585         "/tmp": {"kind": "tmp"},
1586         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1587         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1588         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1589         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1590         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1591     },
1592                 "output_path": "/tmp",
1593                 "priority": 1,
1594                 "runtime_constraints": {},
1595                 "state": "Locked"
1596         }`
1597
1598         extraMounts := []string{
1599                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1600                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1601                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1602         }
1603
1604         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1605                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1606         })
1607
1608         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1609                 "/tmp":                   {realtemp + "/tmp1", false},
1610                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1611                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1612                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1613                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1614         })
1615
1616         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1617         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1618         for _, v := range api.Content {
1619                 if v["collection"] != nil {
1620                         c.Check(v["ensure_unique_name"], Equals, true)
1621                         collection := v["collection"].(arvadosclient.Dict)
1622                         if strings.Index(collection["name"].(string), "output") == 0 {
1623                                 manifest := collection["manifest_text"].(string)
1624
1625                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1626 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1627 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1628 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1629 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1630 `)
1631                         }
1632                 }
1633         }
1634 }
1635
1636 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1637         helperRecord := `{
1638                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1639                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1640                 "cwd": "/bin",
1641                 "environment": {"FROBIZ": "bilbo"},
1642                 "mounts": {
1643         "/tmp": {"kind": "tmp"},
1644         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1645         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1646     },
1647                 "output_path": "/tmp",
1648                 "priority": 1,
1649                 "runtime_constraints": {},
1650                 "state": "Locked"
1651         }`
1652
1653         extraMounts := []string{
1654                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1655         }
1656
1657         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1658                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1659         })
1660
1661         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1662         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1663         for _, v := range s.api.Content {
1664                 if v["collection"] != nil {
1665                         collection := v["collection"].(arvadosclient.Dict)
1666                         if strings.Index(collection["name"].(string), "output") == 0 {
1667                                 manifest := collection["manifest_text"].(string)
1668
1669                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1670 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1671 `)
1672                         }
1673                 }
1674         }
1675 }
1676
1677 func (s *TestSuite) TestOutputError(c *C) {
1678         helperRecord := `{
1679                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1680                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1681                 "cwd": "/bin",
1682                 "environment": {"FROBIZ": "bilbo"},
1683                 "mounts": {
1684                         "/tmp": {"kind": "tmp"}
1685                 },
1686                 "output_path": "/tmp",
1687                 "priority": 1,
1688                 "runtime_constraints": {},
1689                 "state": "Locked"
1690         }`
1691         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1692                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1693         })
1694
1695         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1696 }
1697
1698 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1699         helperRecord := `{
1700                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1701                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1702                 "cwd": "/bin",
1703                 "environment": {"FROBIZ": "bilbo"},
1704                 "mounts": {
1705         "/tmp": {"kind": "tmp"},
1706         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1707         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1708     },
1709                 "output_path": "/tmp",
1710                 "priority": 1,
1711                 "runtime_constraints": {},
1712                 "state": "Locked"
1713         }`
1714
1715         extraMounts := []string{
1716                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1717         }
1718
1719         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1720                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1721         })
1722
1723         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1724         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1725         for _, v := range api.Content {
1726                 if v["collection"] != nil {
1727                         collection := v["collection"].(arvadosclient.Dict)
1728                         if strings.Index(collection["name"].(string), "output") == 0 {
1729                                 manifest := collection["manifest_text"].(string)
1730                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1731 `)
1732                         }
1733                 }
1734         }
1735 }
1736
1737 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1738         helperRecord := `{
1739                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1740                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1741                 "cwd": "/bin",
1742                 "environment": {"FROBIZ": "bilbo"},
1743                 "mounts": {
1744         "/tmp": {"kind": "tmp"},
1745         "stdin": {"kind": "json", "content": "foo"},
1746         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1747     },
1748                 "output_path": "/tmp",
1749                 "priority": 1,
1750                 "runtime_constraints": {},
1751                 "state": "Locked"
1752         }`
1753
1754         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1755                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1756         })
1757
1758         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1759         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1760         for _, v := range api.Content {
1761                 if v["collection"] != nil {
1762                         collection := v["collection"].(arvadosclient.Dict)
1763                         if strings.Index(collection["name"].(string), "output") == 0 {
1764                                 manifest := collection["manifest_text"].(string)
1765                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1766 `)
1767                         }
1768                 }
1769         }
1770 }
1771
1772 func (s *TestSuite) TestStderrMount(c *C) {
1773         api, cr, _ := s.fullRunHelper(c, `{
1774     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1775     "container_image": "`+arvadostest.DockerImage112PDH+`",
1776     "cwd": ".",
1777     "environment": {},
1778     "mounts": {"/tmp": {"kind": "tmp"},
1779                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1780                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1781     "output_path": "/tmp",
1782     "priority": 1,
1783     "runtime_constraints": {},
1784     "state": "Locked"
1785 }`, nil, 1, func() {
1786                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1787                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1788         })
1789
1790         final := api.CalledWith("container.state", "Complete")
1791         c.Assert(final, NotNil)
1792         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1793         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1794
1795         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1796 }
1797
1798 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1799         s.api.callraw = true
1800         err := s.runner.fetchContainerRecord()
1801         c.Assert(err, IsNil)
1802         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1803         c.Logf("%#v", s.runner.Container)
1804         c.Check(err, IsNil)
1805         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1806 }
1807
1808 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1809         nextState := ""
1810         for _, setup := range []func(){
1811                 func() {
1812                         c.Log("// waitErr = ocl runtime error")
1813                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1814                         nextState = "Cancelled"
1815                 },
1816                 func() {
1817                         c.Log("// loadErr = cannot connect")
1818                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1819                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1820                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1821                         c.Assert(err, IsNil)
1822                         nextState = "Queued"
1823                 },
1824         } {
1825                 s.SetUpTest(c)
1826                 setup()
1827                 s.fullRunHelper(c, `{
1828     "command": ["echo", "hello world"],
1829     "container_image": "`+arvadostest.DockerImage112PDH+`",
1830     "cwd": ".",
1831     "environment": {},
1832     "mounts": {"/tmp": {"kind": "tmp"} },
1833     "output_path": "/tmp",
1834     "priority": 1,
1835     "runtime_constraints": {},
1836     "state": "Locked"
1837 }`, nil, 0, func() {})
1838                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1839                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1840                 if *brokenNodeHook != "" {
1841                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1842                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1843                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1844                 } else {
1845                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1846                 }
1847         }
1848 }
1849
1850 func (s *TestSuite) TestBadCommand(c *C) {
1851         for _, startError := range []string{
1852                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1853                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1854                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1855         } {
1856                 s.SetUpTest(c)
1857                 s.executor.startErr = errors.New(startError)
1858                 s.fullRunHelper(c, `{
1859     "command": ["echo", "hello world"],
1860     "container_image": "`+arvadostest.DockerImage112PDH+`",
1861     "cwd": ".",
1862     "environment": {},
1863     "mounts": {"/tmp": {"kind": "tmp"} },
1864     "output_path": "/tmp",
1865     "priority": 1,
1866     "runtime_constraints": {},
1867     "state": "Locked"
1868 }`, nil, 0, func() {})
1869                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1870                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1871         }
1872 }
1873
1874 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1875         helperRecord := `{
1876                 "command": ["true"],
1877                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1878                 "cwd": "/bin",
1879                 "mounts": {
1880                     "/tmp": {"kind": "tmp"},
1881                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1882                 },
1883                 "secret_mounts": {
1884                 },
1885                 "output_path": "/tmp",
1886                 "priority": 1,
1887                 "runtime_constraints": {},
1888                 "state": "Locked"
1889         }`
1890
1891         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1892                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1893                 c.Check(err, IsNil)
1894                 c.Check(string(content), Equals, "mypassword")
1895         })
1896
1897         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1898         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1899         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1900         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1901
1902         // under secret mounts, not captured in output
1903         helperRecord = `{
1904                 "command": ["true"],
1905                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1906                 "cwd": "/bin",
1907                 "mounts": {
1908                     "/tmp": {"kind": "tmp"}
1909                 },
1910                 "secret_mounts": {
1911                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1912                 },
1913                 "output_path": "/tmp",
1914                 "priority": 1,
1915                 "runtime_constraints": {},
1916                 "state": "Locked"
1917         }`
1918
1919         s.SetUpTest(c)
1920         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1921                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1922                 c.Check(err, IsNil)
1923                 c.Check(string(content), Equals, "mypassword")
1924         })
1925
1926         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1927         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1928         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1929         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1930 }
1931
1932 type FakeProcess struct {
1933         cmdLine []string
1934 }
1935
1936 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1937         return fp.cmdLine, nil
1938 }