lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
 116 func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
 117 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 118 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 119 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 120 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 121 func (e *stubExecutor) Close()                          { e.closed = true }
 122 func (e *stubExecutor) Wait(context.Context) (int, error) {
 123         defer e.created.Stdout.Close()
 124         defer e.created.Stderr.Close()
 125         return <-e.exit, e.waitErr
 126 }
 127
 128 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 129
 130 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 131 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 132 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 133
 134 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 135 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 136
 137 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 138 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 139 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 140 `
 141
 142 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 143
 144 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 145 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 146
 147 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 148 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 149
 150 func (client *ArvTestClient) Create(resourceType string,
 151         parameters arvadosclient.Dict,
 152         output interface{}) error {
 153
 154         client.Mutex.Lock()
 155         defer client.Mutex.Unlock()
 156
 157         client.Calls++
 158         client.Content = append(client.Content, parameters)
 159
 160         if resourceType == "logs" {
 161                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 162                 if client.Logs == nil {
 163                         client.Logs = make(map[string]*bytes.Buffer)
 164                 }
 165                 if client.Logs[et] == nil {
 166                         client.Logs[et] = &bytes.Buffer{}
 167                 }
 168                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 169         }
 170
 171         if resourceType == "collections" && output != nil {
 172                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 173                 outmap := output.(*arvados.Collection)
 174                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 175                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 176         }
 177
 178         return nil
 179 }
 180
 181 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 182         switch {
 183         case method == "GET" && resourceType == "containers" && action == "auth":
 184                 return json.Unmarshal([]byte(`{
 185                         "kind": "arvados#api_client_authorization",
 186                         "uuid": "`+fakeAuthUUID+`",
 187                         "api_token": "`+fakeAuthToken+`"
 188                         }`), output)
 189         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 190                 if client.secretMounts != nil {
 191                         return json.Unmarshal(client.secretMounts, output)
 192                 }
 193                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 194         default:
 195                 return fmt.Errorf("Not found")
 196         }
 197 }
 198
 199 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 200         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 201         var j []byte
 202         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 203                 j = []byte(`{
 204                         "kind": "arvados#nodeList",
 205                         "items": [{
 206                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 207                                 "hostname": "compute2",
 208                                 "properties": {"total_cpu_cores": 16}
 209                         }]}`)
 210         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 211                 if uuid == "" {
 212                         j, err = json.Marshal(map[string]interface{}{
 213                                 "items": []interface{}{client.Container},
 214                                 "kind":  "arvados#nodeList",
 215                         })
 216                 } else {
 217                         j, err = json.Marshal(client.Container)
 218                 }
 219         } else {
 220                 j = []byte(`{
 221                         "command": ["sleep", "1"],
 222                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 223                         "cwd": ".",
 224                         "environment": {},
 225                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 226                         "output_path": "/tmp",
 227                         "priority": 1,
 228                         "runtime_constraints": {}
 229                 }`)
 230         }
 231         return ioutil.NopCloser(bytes.NewReader(j)), err
 232 }
 233
 234 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 235         if resourceType == "collections" {
 236                 if uuid == hwPDH {
 237                         output.(*arvados.Collection).ManifestText = hwManifest
 238                 } else if uuid == otherPDH {
 239                         output.(*arvados.Collection).ManifestText = otherManifest
 240                 } else if uuid == normalizedWithSubdirsPDH {
 241                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 242                 } else if uuid == denormalizedWithSubdirsPDH {
 243                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 244                 }
 245         }
 246         if resourceType == "containers" {
 247                 (*output.(*arvados.Container)) = client.Container
 248         }
 249         return nil
 250 }
 251
 252 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 253         client.Mutex.Lock()
 254         defer client.Mutex.Unlock()
 255         client.Calls++
 256         client.Content = append(client.Content, parameters)
 257         if resourceType == "containers" {
 258                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 259                         client.WasSetRunning = true
 260                 }
 261         } else if resourceType == "collections" {
 262                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 263                 output.(*arvados.Collection).UUID = uuid
 264                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 265         }
 266         return nil
 267 }
 268
 269 var discoveryMap = map[string]interface{}{
 270         "defaultTrashLifetime":               float64(1209600),
 271         "crunchLimitLogBytesPerJob":          float64(67108864),
 272         "crunchLogThrottleBytes":             float64(65536),
 273         "crunchLogThrottlePeriod":            float64(60),
 274         "crunchLogThrottleLines":             float64(1024),
 275         "crunchLogPartialLineThrottlePeriod": float64(5),
 276         "crunchLogBytesPerEvent":             float64(4096),
 277         "crunchLogSecondsBetweenEvents":      float64(1),
 278 }
 279
 280 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 281         return discoveryMap[key], nil
 282 }
 283
 284 // CalledWith returns the parameters from the first API call whose
 285 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 286 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 287 // no call matches, it returns nil.
 288 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 289 call:
 290         for _, content := range client.Content {
 291                 var v interface{} = content
 292                 for _, k := range strings.Split(jpath, ".") {
 293                         if dict, ok := v.(arvadosclient.Dict); !ok {
 294                                 continue call
 295                         } else {
 296                                 v = dict[k]
 297                         }
 298                 }
 299                 if v == expect {
 300                         return content
 301                 }
 302         }
 303         return nil
 304 }
 305
 306 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 307         return locator, nil
 308 }
 309
 310 func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
 311         client.Content = buf
 312         return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
 313 }
 314
 315 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 316         return 0, errors.New("not implemented")
 317 }
 318
 319 func (client *KeepTestClient) ClearBlockCache() {
 320 }
 321
 322 func (client *KeepTestClient) Close() {
 323         client.Content = nil
 324 }
 325
 326 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 327         client.StorageClasses = sc
 328 }
 329
 330 type FileWrapper struct {
 331         io.ReadCloser
 332         len int64
 333 }
 334
 335 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 336         return nil, errors.New("not implemented")
 337 }
 338
 339 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 340         return 0, errors.New("not implemented")
 341 }
 342
 343 func (fw FileWrapper) Size() int64 {
 344         return fw.len
 345 }
 346
 347 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 348         return nil, errors.New("not implemented")
 349 }
 350
 351 func (fw FileWrapper) Truncate(int64) error {
 352         return errors.New("not implemented")
 353 }
 354
 355 func (fw FileWrapper) Write([]byte) (int, error) {
 356         return 0, errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Sync() error {
 360         return errors.New("not implemented")
 361 }
 362
 363 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 364         if filename == hwImageID+".tar" {
 365                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 366                 client.Called = true
 367                 return FileWrapper{rdr, 1321984}, nil
 368         } else if filename == "/file1_in_main.txt" {
 369                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 370                 client.Called = true
 371                 return FileWrapper{rdr, 3}, nil
 372         }
 373         return nil, nil
 374 }
 375
 376 func (s *TestSuite) TestLoadImage(c *C) {
 377         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 378         s.runner.Container.Mounts = map[string]arvados.Mount{
 379                 "/out": {Kind: "tmp", Writable: true},
 380         }
 381         s.runner.Container.OutputPath = "/out"
 382
 383         _, err := s.runner.SetupMounts()
 384         c.Assert(err, IsNil)
 385
 386         imageID, err := s.runner.LoadImage()
 387         c.Check(err, IsNil)
 388         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 389         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 390
 391         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 392         s.executor.imageLoaded = false
 393         s.executor.loaded = ""
 394         s.executor.loadErr = errors.New("bork")
 395         imageID, err = s.runner.LoadImage()
 396         c.Check(err, ErrorMatches, ".*bork")
 397         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 398
 399         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 400         s.executor.imageLoaded = false
 401         s.executor.loaded = ""
 402         s.executor.loadErr = nil
 403         imageID, err = s.runner.LoadImage()
 404         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 405         c.Check(s.executor.loaded, Equals, "")
 406
 407         // if executor reports image is already loaded, LoadImage should not be called
 408         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 409         s.executor.imageLoaded = true
 410         s.executor.loaded = ""
 411         s.executor.loadErr = nil
 412         imageID, err = s.runner.LoadImage()
 413         c.Check(err, IsNil)
 414         c.Check(s.executor.loaded, Equals, "")
 415         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 416 }
 417
 418 type ArvErrorTestClient struct{}
 419
 420 func (ArvErrorTestClient) Create(resourceType string,
 421         parameters arvadosclient.Dict,
 422         output interface{}) error {
 423         return nil
 424 }
 425
 426 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 427         if method == "GET" && resourceType == "containers" && action == "auth" {
 428                 return nil
 429         }
 430         return errors.New("ArvError")
 431 }
 432
 433 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 434         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 435         return nil, errors.New("ArvError")
 436 }
 437
 438 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 439         return errors.New("ArvError")
 440 }
 441
 442 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 443         return nil
 444 }
 445
 446 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 447         return discoveryMap[key], nil
 448 }
 449
 450 type KeepErrorTestClient struct {
 451         KeepTestClient
 452 }
 453
 454 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 455         return nil, errors.New("KeepError")
 456 }
 457
 458 func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
 459         return "", 0, errors.New("KeepError")
 460 }
 461
 462 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 463         return "", errors.New("KeepError")
 464 }
 465
 466 type KeepReadErrorTestClient struct {
 467         KeepTestClient
 468 }
 469
 470 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 471         return 0, errors.New("KeepError")
 472 }
 473
 474 type ErrorReader struct {
 475         FileWrapper
 476 }
 477
 478 func (ErrorReader) Read(p []byte) (n int, err error) {
 479         return 0, errors.New("ErrorReader")
 480 }
 481
 482 func (ErrorReader) Seek(int64, int) (int64, error) {
 483         return 0, errors.New("ErrorReader")
 484 }
 485
 486 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 487         return ErrorReader{}, nil
 488 }
 489
 490 type ClosableBuffer struct {
 491         bytes.Buffer
 492 }
 493
 494 func (*ClosableBuffer) Close() error {
 495         return nil
 496 }
 497
 498 type TestLogs struct {
 499         Stdout ClosableBuffer
 500         Stderr ClosableBuffer
 501 }
 502
 503 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 504         if logstr == "stdout" {
 505                 return &tl.Stdout, nil
 506         }
 507         if logstr == "stderr" {
 508                 return &tl.Stderr, nil
 509         }
 510         return nil, errors.New("???")
 511 }
 512
 513 func dockerLog(fd byte, msg string) []byte {
 514         by := []byte(msg)
 515         header := make([]byte, 8+len(by))
 516         header[0] = fd
 517         header[7] = byte(len(by))
 518         copy(header[8:], by)
 519         return header
 520 }
 521
 522 func (s *TestSuite) TestRunContainer(c *C) {
 523         s.executor.runFunc = func() {
 524                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 525                 s.executor.created.Stdout.Close()
 526                 s.executor.created.Stderr.Close()
 527                 s.executor.exit <- 0
 528         }
 529
 530         var logs TestLogs
 531         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 532         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 533         s.runner.Container.Command = []string{"./hw"}
 534         s.runner.Container.OutputStorageClasses = []string{"default"}
 535
 536         imageID, err := s.runner.LoadImage()
 537         c.Assert(err, IsNil)
 538
 539         err = s.runner.CreateContainer(imageID, nil)
 540         c.Assert(err, IsNil)
 541
 542         err = s.runner.StartContainer()
 543         c.Assert(err, IsNil)
 544
 545         err = s.runner.WaitFinish()
 546         c.Assert(err, IsNil)
 547
 548         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 549         c.Check(logs.Stderr.String(), Equals, "")
 550 }
 551
 552 func (s *TestSuite) TestCommitLogs(c *C) {
 553         api := &ArvTestClient{}
 554         kc := &KeepTestClient{}
 555         defer kc.Close()
 556         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 557         c.Assert(err, IsNil)
 558         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 559
 560         cr.CrunchLog.Print("Hello world!")
 561         cr.CrunchLog.Print("Goodbye")
 562         cr.finalState = "Complete"
 563
 564         err = cr.CommitLogs()
 565         c.Check(err, IsNil)
 566
 567         c.Check(api.Calls, Equals, 2)
 568         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 569         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 570         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 571         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 572 }
 573
 574 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 575         api := &ArvTestClient{}
 576         kc := &KeepTestClient{}
 577         defer kc.Close()
 578         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 579         c.Assert(err, IsNil)
 580
 581         err = cr.UpdateContainerRunning()
 582         c.Check(err, IsNil)
 583
 584         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 585 }
 586
 587 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 588         api := &ArvTestClient{}
 589         kc := &KeepTestClient{}
 590         defer kc.Close()
 591         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 592         c.Assert(err, IsNil)
 593
 594         cr.LogsPDH = new(string)
 595         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 596
 597         cr.ExitCode = new(int)
 598         *cr.ExitCode = 42
 599         cr.finalState = "Complete"
 600
 601         err = cr.UpdateContainerFinal()
 602         c.Check(err, IsNil)
 603
 604         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 605         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 606         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 607 }
 608
 609 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 610         api := &ArvTestClient{}
 611         kc := &KeepTestClient{}
 612         defer kc.Close()
 613         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 614         c.Assert(err, IsNil)
 615         cr.cCancelled = true
 616         cr.finalState = "Cancelled"
 617
 618         err = cr.UpdateContainerFinal()
 619         c.Check(err, IsNil)
 620
 621         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 622         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 624 }
 625
 626 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 627 // dress rehearsal of the Run() function, starting from a JSON container record.
 628 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 629         err := json.Unmarshal([]byte(record), &s.api.Container)
 630         c.Assert(err, IsNil)
 631         initialState := s.api.Container.State
 632
 633         var sm struct {
 634                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 635         }
 636         err = json.Unmarshal([]byte(record), &sm)
 637         c.Check(err, IsNil)
 638         secretMounts, err := json.Marshal(sm)
 639         c.Assert(err, IsNil)
 640         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 641
 642         s.executor.runFunc = func() {
 643                 fn()
 644                 s.executor.exit <- exitCode
 645         }
 646
 647         s.runner.statInterval = 100 * time.Millisecond
 648         s.runner.containerWatchdogInterval = time.Second
 649         am := &ArvMountCmdLine{}
 650         s.runner.RunArvMount = am.ArvMountTest
 651
 652         realTemp := c.MkDir()
 653         tempcount := 0
 654         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 655                 tempcount++
 656                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 657                 err := os.Mkdir(d, os.ModePerm)
 658                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 659                         // Test case must have pre-populated the tempdir
 660                         err = nil
 661                 }
 662                 return d, err
 663         }
 664         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 665                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 666         }
 667
 668         if extraMounts != nil && len(extraMounts) > 0 {
 669                 err := s.runner.SetupArvMountPoint("keep")
 670                 c.Check(err, IsNil)
 671
 672                 for _, m := range extraMounts {
 673                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 674                 }
 675         }
 676
 677         err = s.runner.Run()
 678         if s.api.CalledWith("container.state", "Complete") != nil {
 679                 c.Check(err, IsNil)
 680         }
 681         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 682                 c.Check(s.api.WasSetRunning, Equals, true)
 683                 var lastupdate arvadosclient.Dict
 684                 for _, content := range s.api.Content {
 685                         if content["container"] != nil {
 686                                 lastupdate = content["container"].(arvadosclient.Dict)
 687                         }
 688                 }
 689                 if lastupdate["log"] == nil {
 690                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 691                 }
 692         }
 693
 694         if err != nil {
 695                 for k, v := range s.api.Logs {
 696                         c.Log(k)
 697                         c.Log(v.String())
 698                 }
 699         }
 700
 701         return s.api, s.runner, realTemp
 702 }
 703
 704 func (s *TestSuite) TestFullRunHello(c *C) {
 705         s.runner.enableMemoryLimit = true
 706         s.runner.networkMode = "default"
 707         s.fullRunHelper(c, `{
 708     "command": ["echo", "hello world"],
 709     "container_image": "`+arvadostest.DockerImage112PDH+`",
 710     "cwd": ".",
 711     "environment": {"foo":"bar","baz":"waz"},
 712     "mounts": {"/tmp": {"kind": "tmp"} },
 713     "output_path": "/tmp",
 714     "priority": 1,
 715     "runtime_constraints": {"vcpus":1,"ram":1000000},
 716     "state": "Locked",
 717     "output_storage_classes": ["default"]
 718 }`, nil, 0, func() {
 719                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 720                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 721                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 722                 c.Check(s.executor.created.VCPUs, Equals, 1)
 723                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 724                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 725                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 726                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 727         })
 728
 729         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 730         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 731         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 732         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 733         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 734 }
 735
 736 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 737         var ran bool
 738         s.fullRunHelper(c, `{
 739     "command": ["sleep", "3"],
 740     "container_image": "`+arvadostest.DockerImage112PDH+`",
 741     "cwd": ".",
 742     "environment": {},
 743     "mounts": {"/tmp": {"kind": "tmp"} },
 744     "output_path": "/tmp",
 745     "priority": 1,
 746     "runtime_constraints": {},
 747     "scheduling_parameters":{"max_run_time": 1},
 748     "state": "Running"
 749 }`, nil, 2, func() {
 750                 ran = true
 751         })
 752         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 753         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 754         c.Check(ran, Equals, false)
 755 }
 756
 757 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 758         s.fullRunHelper(c, `{
 759     "command": ["sleep", "3"],
 760     "container_image": "`+arvadostest.DockerImage112PDH+`",
 761     "cwd": ".",
 762     "environment": {},
 763     "mounts": {"/tmp": {"kind": "tmp"} },
 764     "output_path": "/tmp",
 765     "priority": 1,
 766     "runtime_constraints": {},
 767     "scheduling_parameters":{"max_run_time": 1},
 768     "state": "Locked"
 769 }`, nil, 0, func() {
 770                 time.Sleep(3 * time.Second)
 771         })
 772
 773         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 774         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 775 }
 776
 777 func (s *TestSuite) TestContainerWaitFails(c *C) {
 778         s.fullRunHelper(c, `{
 779     "command": ["sleep", "3"],
 780     "container_image": "`+arvadostest.DockerImage112PDH+`",
 781     "cwd": ".",
 782     "mounts": {"/tmp": {"kind": "tmp"} },
 783     "output_path": "/tmp",
 784     "priority": 1,
 785     "state": "Locked"
 786 }`, nil, 0, func() {
 787                 s.executor.waitErr = errors.New("Container is not running")
 788         })
 789
 790         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 791         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 792 }
 793
 794 func (s *TestSuite) TestCrunchstat(c *C) {
 795         s.fullRunHelper(c, `{
 796                 "command": ["sleep", "1"],
 797                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 798                 "cwd": ".",
 799                 "environment": {},
 800                 "mounts": {"/tmp": {"kind": "tmp"} },
 801                 "output_path": "/tmp",
 802                 "priority": 1,
 803                 "runtime_constraints": {},
 804                 "state": "Locked"
 805         }`, nil, 0, func() {
 806                 time.Sleep(time.Second)
 807         })
 808
 809         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 810         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 811
 812         // We didn't actually start a container, so crunchstat didn't
 813         // find accounting files and therefore didn't log any stats.
 814         // It should have logged a "can't find accounting files"
 815         // message after one poll interval, though, so we can confirm
 816         // it's alive:
 817         c.Assert(s.api.Logs["crunchstat"], NotNil)
 818         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 819
 820         // The "files never appeared" log assures us that we called
 821         // (*crunchstat.Reporter)Stop(), and that we set it up with
 822         // the correct container ID "abcde":
 823         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 824 }
 825
 826 func (s *TestSuite) TestNodeInfoLog(c *C) {
 827         os.Setenv("SLURMD_NODENAME", "compute2")
 828         s.fullRunHelper(c, `{
 829                 "command": ["sleep", "1"],
 830                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 831                 "cwd": ".",
 832                 "environment": {},
 833                 "mounts": {"/tmp": {"kind": "tmp"} },
 834                 "output_path": "/tmp",
 835                 "priority": 1,
 836                 "runtime_constraints": {},
 837                 "state": "Locked"
 838         }`, nil, 0,
 839                 func() {
 840                         time.Sleep(time.Second)
 841                 })
 842
 843         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 844         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 845
 846         c.Assert(s.api.Logs["node"], NotNil)
 847         json := s.api.Logs["node"].String()
 848         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 849         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 850         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 851
 852         c.Assert(s.api.Logs["node-info"], NotNil)
 853         json = s.api.Logs["node-info"].String()
 854         c.Check(json, Matches, `(?ms).*Host Information.*`)
 855         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 856         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 857         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 858         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 859 }
 860
 861 func (s *TestSuite) TestContainerRecordLog(c *C) {
 862         s.fullRunHelper(c, `{
 863                 "command": ["sleep", "1"],
 864                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 865                 "cwd": ".",
 866                 "environment": {},
 867                 "mounts": {"/tmp": {"kind": "tmp"} },
 868                 "output_path": "/tmp",
 869                 "priority": 1,
 870                 "runtime_constraints": {},
 871                 "state": "Locked"
 872         }`, nil, 0,
 873                 func() {
 874                         time.Sleep(time.Second)
 875                 })
 876
 877         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 878         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 879
 880         c.Assert(s.api.Logs["container"], NotNil)
 881         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 882 }
 883
 884 func (s *TestSuite) TestFullRunStderr(c *C) {
 885         s.fullRunHelper(c, `{
 886     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 887     "container_image": "`+arvadostest.DockerImage112PDH+`",
 888     "cwd": ".",
 889     "environment": {},
 890     "mounts": {"/tmp": {"kind": "tmp"} },
 891     "output_path": "/tmp",
 892     "priority": 1,
 893     "runtime_constraints": {},
 894     "state": "Locked"
 895 }`, nil, 1, func() {
 896                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 897                 fmt.Fprintln(s.executor.created.Stderr, "world")
 898         })
 899
 900         final := s.api.CalledWith("container.state", "Complete")
 901         c.Assert(final, NotNil)
 902         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 903         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 904
 905         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 906         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 907 }
 908
 909 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 910         s.fullRunHelper(c, `{
 911     "command": ["pwd"],
 912     "container_image": "`+arvadostest.DockerImage112PDH+`",
 913     "cwd": ".",
 914     "environment": {},
 915     "mounts": {"/tmp": {"kind": "tmp"} },
 916     "output_path": "/tmp",
 917     "priority": 1,
 918     "runtime_constraints": {},
 919     "state": "Locked"
 920 }`, nil, 0, func() {
 921                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 922         })
 923
 924         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 925         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 926         c.Log(s.api.Logs["stdout"])
 927         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 928 }
 929
 930 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 931         s.fullRunHelper(c, `{
 932     "command": ["pwd"],
 933     "container_image": "`+arvadostest.DockerImage112PDH+`",
 934     "cwd": "/bin",
 935     "environment": {},
 936     "mounts": {"/tmp": {"kind": "tmp"} },
 937     "output_path": "/tmp",
 938     "priority": 1,
 939     "runtime_constraints": {},
 940     "state": "Locked"
 941 }`, nil, 0, func() {
 942                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 943         })
 944
 945         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 946         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 947         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 948 }
 949
 950 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 951         s.fullRunHelper(c, `{
 952     "command": ["pwd"],
 953     "container_image": "`+arvadostest.DockerImage112PDH+`",
 954     "cwd": "/bin",
 955     "environment": {},
 956     "mounts": {"/tmp": {"kind": "tmp"} },
 957     "output_path": "/tmp",
 958     "priority": 1,
 959     "runtime_constraints": {},
 960     "state": "Locked",
 961     "output_storage_classes": ["foo", "bar"]
 962 }`, nil, 0, func() {
 963                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 964         })
 965
 966         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 967         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 968         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 969         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 970         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 971 }
 972
 973 func (s *TestSuite) TestStopOnSignal(c *C) {
 974         s.executor.runFunc = func() {
 975                 s.executor.created.Stdout.Write([]byte("foo\n"))
 976                 s.runner.SigChan <- syscall.SIGINT
 977         }
 978         s.testStopContainer(c)
 979 }
 980
 981 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 982         s.executor.runFunc = func() {
 983                 s.executor.created.Stdout.Write([]byte("foo\n"))
 984                 s.runner.ArvMountExit <- nil
 985                 close(s.runner.ArvMountExit)
 986         }
 987         s.runner.ArvMountExit = make(chan error)
 988         s.testStopContainer(c)
 989 }
 990
 991 func (s *TestSuite) testStopContainer(c *C) {
 992         record := `{
 993     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 994     "container_image": "` + arvadostest.DockerImage112PDH + `",
 995     "cwd": ".",
 996     "environment": {},
 997     "mounts": {"/tmp": {"kind": "tmp"} },
 998     "output_path": "/tmp",
 999     "priority": 1,
1000     "runtime_constraints": {},
1001     "state": "Locked"
1002 }`
1003
1004         err := json.Unmarshal([]byte(record), &s.api.Container)
1005         c.Assert(err, IsNil)
1006
1007         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1008         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1009                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1010         }
1011
1012         done := make(chan error)
1013         go func() {
1014                 done <- s.runner.Run()
1015         }()
1016         select {
1017         case <-time.After(20 * time.Second):
1018                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1019                 c.Fatal("timed out")
1020         case err = <-done:
1021                 c.Check(err, IsNil)
1022         }
1023         for k, v := range s.api.Logs {
1024                 c.Log(k)
1025                 c.Log(v.String(), "\n")
1026         }
1027
1028         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1029         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1030         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1031 }
1032
1033 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1034         s.fullRunHelper(c, `{
1035     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1036     "container_image": "`+arvadostest.DockerImage112PDH+`",
1037     "cwd": "/bin",
1038     "environment": {"FROBIZ": "bilbo"},
1039     "mounts": {"/tmp": {"kind": "tmp"} },
1040     "output_path": "/tmp",
1041     "priority": 1,
1042     "runtime_constraints": {},
1043     "state": "Locked"
1044 }`, nil, 0, func() {
1045                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1046         })
1047
1048         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1049         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1050         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1051 }
1052
1053 type ArvMountCmdLine struct {
1054         Cmd   []string
1055         token string
1056 }
1057
1058 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1059         am.Cmd = c
1060         am.token = token
1061         return nil, nil
1062 }
1063
1064 func stubCert(temp string) string {
1065         path := temp + "/ca-certificates.crt"
1066         crt, _ := os.Create(path)
1067         crt.Close()
1068         arvadosclient.CertFiles = []string{path}
1069         return path
1070 }
1071
1072 func (s *TestSuite) TestSetupMounts(c *C) {
1073         cr := s.runner
1074         am := &ArvMountCmdLine{}
1075         cr.RunArvMount = am.ArvMountTest
1076         cr.ContainerArvClient = &ArvTestClient{}
1077         cr.ContainerKeepClient = &KeepTestClient{}
1078         cr.Container.OutputStorageClasses = []string{"default"}
1079
1080         realTemp := c.MkDir()
1081         certTemp := c.MkDir()
1082         stubCertPath := stubCert(certTemp)
1083         cr.parentTemp = realTemp
1084
1085         i := 0
1086         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1087                 i++
1088                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1089                 err := os.Mkdir(d, os.ModePerm)
1090                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1091                         // Test case must have pre-populated the tempdir
1092                         err = nil
1093                 }
1094                 return d, err
1095         }
1096
1097         checkEmpty := func() {
1098                 // Should be deleted.
1099                 _, err := os.Stat(realTemp)
1100                 c.Assert(os.IsNotExist(err), Equals, true)
1101
1102                 // Now recreate it for the next test.
1103                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1104         }
1105
1106         {
1107                 i = 0
1108                 cr.ArvMountPoint = ""
1109                 cr.Container.Mounts = make(map[string]arvados.Mount)
1110                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1111                 cr.Container.OutputPath = "/tmp"
1112                 cr.statInterval = 5 * time.Second
1113                 bindmounts, err := cr.SetupMounts()
1114                 c.Check(err, IsNil)
1115                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1116                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1117                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1118                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1119                 os.RemoveAll(cr.ArvMountPoint)
1120                 cr.CleanupDirs()
1121                 checkEmpty()
1122         }
1123
1124         {
1125                 i = 0
1126                 cr.ArvMountPoint = ""
1127                 cr.Container.Mounts = make(map[string]arvados.Mount)
1128                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1129                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1130                 cr.Container.OutputPath = "/out"
1131                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1132
1133                 bindmounts, err := cr.SetupMounts()
1134                 c.Check(err, IsNil)
1135                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1136                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1137                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1138                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1139                 os.RemoveAll(cr.ArvMountPoint)
1140                 cr.CleanupDirs()
1141                 checkEmpty()
1142         }
1143
1144         {
1145                 i = 0
1146                 cr.ArvMountPoint = ""
1147                 cr.Container.Mounts = make(map[string]arvados.Mount)
1148                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1149                 cr.Container.OutputPath = "/tmp"
1150                 cr.Container.RuntimeConstraints.API = true
1151                 cr.Container.OutputStorageClasses = []string{"default"}
1152
1153                 bindmounts, err := cr.SetupMounts()
1154                 c.Check(err, IsNil)
1155                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1156                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1157                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1158                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1159                 os.RemoveAll(cr.ArvMountPoint)
1160                 cr.CleanupDirs()
1161                 checkEmpty()
1162
1163                 cr.Container.RuntimeConstraints.API = false
1164         }
1165
1166         {
1167                 i = 0
1168                 cr.ArvMountPoint = ""
1169                 cr.Container.Mounts = map[string]arvados.Mount{
1170                         "/keeptmp": {Kind: "collection", Writable: true},
1171                 }
1172                 cr.Container.OutputPath = "/keeptmp"
1173
1174                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1175
1176                 bindmounts, err := cr.SetupMounts()
1177                 c.Check(err, IsNil)
1178                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1179                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1180                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1181                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1182                 os.RemoveAll(cr.ArvMountPoint)
1183                 cr.CleanupDirs()
1184                 checkEmpty()
1185         }
1186
1187         {
1188                 i = 0
1189                 cr.ArvMountPoint = ""
1190                 cr.Container.Mounts = map[string]arvados.Mount{
1191                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1192                         "/keepout": {Kind: "collection", Writable: true},
1193                 }
1194                 cr.Container.OutputPath = "/keepout"
1195
1196                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1197                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1198
1199                 bindmounts, err := cr.SetupMounts()
1200                 c.Check(err, IsNil)
1201                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1202                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1203                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1204                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1205                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1206                         "/keepout": {realTemp + "/keep1/tmp0", false},
1207                 })
1208                 os.RemoveAll(cr.ArvMountPoint)
1209                 cr.CleanupDirs()
1210                 checkEmpty()
1211         }
1212
1213         {
1214                 i = 0
1215                 cr.ArvMountPoint = ""
1216                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1217                 cr.Container.Mounts = map[string]arvados.Mount{
1218                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1219                         "/keepout": {Kind: "collection", Writable: true},
1220                 }
1221                 cr.Container.OutputPath = "/keepout"
1222
1223                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1224                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1225
1226                 bindmounts, err := cr.SetupMounts()
1227                 c.Check(err, IsNil)
1228                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1229                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1230                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1231                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1232                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1233                         "/keepout": {realTemp + "/keep1/tmp0", false},
1234                 })
1235                 os.RemoveAll(cr.ArvMountPoint)
1236                 cr.CleanupDirs()
1237                 checkEmpty()
1238         }
1239
1240         for _, test := range []struct {
1241                 in  interface{}
1242                 out string
1243         }{
1244                 {in: "foo", out: `"foo"`},
1245                 {in: nil, out: `null`},
1246                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1247         } {
1248                 i = 0
1249                 cr.ArvMountPoint = ""
1250                 cr.Container.Mounts = map[string]arvados.Mount{
1251                         "/mnt/test.json": {Kind: "json", Content: test.in},
1252                 }
1253                 bindmounts, err := cr.SetupMounts()
1254                 c.Check(err, IsNil)
1255                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1256                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1257                 })
1258                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1259                 c.Check(err, IsNil)
1260                 c.Check(content, DeepEquals, []byte(test.out))
1261                 os.RemoveAll(cr.ArvMountPoint)
1262                 cr.CleanupDirs()
1263                 checkEmpty()
1264         }
1265
1266         for _, test := range []struct {
1267                 in  interface{}
1268                 out string
1269         }{
1270                 {in: "foo", out: `foo`},
1271                 {in: nil, out: "error"},
1272                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1273         } {
1274                 i = 0
1275                 cr.ArvMountPoint = ""
1276                 cr.Container.Mounts = map[string]arvados.Mount{
1277                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1278                 }
1279                 bindmounts, err := cr.SetupMounts()
1280                 if test.out == "error" {
1281                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1282                 } else {
1283                         c.Check(err, IsNil)
1284                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1285                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1286                         })
1287                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1288                         c.Check(err, IsNil)
1289                         c.Check(content, DeepEquals, []byte(test.out))
1290                 }
1291                 os.RemoveAll(cr.ArvMountPoint)
1292                 cr.CleanupDirs()
1293                 checkEmpty()
1294         }
1295
1296         // Read-only mount points are allowed underneath output_dir mount point
1297         {
1298                 i = 0
1299                 cr.ArvMountPoint = ""
1300                 cr.Container.Mounts = make(map[string]arvados.Mount)
1301                 cr.Container.Mounts = map[string]arvados.Mount{
1302                         "/tmp":     {Kind: "tmp"},
1303                         "/tmp/foo": {Kind: "collection"},
1304                 }
1305                 cr.Container.OutputPath = "/tmp"
1306
1307                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1308
1309                 bindmounts, err := cr.SetupMounts()
1310                 c.Check(err, IsNil)
1311                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1312                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1313                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1314                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1315                         "/tmp":     {realTemp + "/tmp2", false},
1316                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1317                 })
1318                 os.RemoveAll(cr.ArvMountPoint)
1319                 cr.CleanupDirs()
1320                 checkEmpty()
1321         }
1322
1323         // Writable mount points copied to output_dir mount point
1324         {
1325                 i = 0
1326                 cr.ArvMountPoint = ""
1327                 cr.Container.Mounts = make(map[string]arvados.Mount)
1328                 cr.Container.Mounts = map[string]arvados.Mount{
1329                         "/tmp": {Kind: "tmp"},
1330                         "/tmp/foo": {Kind: "collection",
1331                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1332                                 Writable:         true},
1333                         "/tmp/bar": {Kind: "collection",
1334                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1335                                 Path:             "baz",
1336                                 Writable:         true},
1337                 }
1338                 cr.Container.OutputPath = "/tmp"
1339
1340                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1341                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1342
1343                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1344                 rf.Write([]byte("bar"))
1345                 rf.Close()
1346
1347                 _, err := cr.SetupMounts()
1348                 c.Check(err, IsNil)
1349                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1350                 c.Check(err, IsNil)
1351                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1352                 c.Check(err, IsNil)
1353                 os.RemoveAll(cr.ArvMountPoint)
1354                 cr.CleanupDirs()
1355                 checkEmpty()
1356         }
1357
1358         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1359         {
1360                 i = 0
1361                 cr.ArvMountPoint = ""
1362                 cr.Container.Mounts = make(map[string]arvados.Mount)
1363                 cr.Container.Mounts = map[string]arvados.Mount{
1364                         "/tmp":     {Kind: "tmp"},
1365                         "/tmp/foo": {Kind: "tmp"},
1366                 }
1367                 cr.Container.OutputPath = "/tmp"
1368
1369                 _, err := cr.SetupMounts()
1370                 c.Check(err, NotNil)
1371                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1372                 os.RemoveAll(cr.ArvMountPoint)
1373                 cr.CleanupDirs()
1374                 checkEmpty()
1375         }
1376
1377         // Only mount point of kind 'collection' is allowed for stdin
1378         {
1379                 i = 0
1380                 cr.ArvMountPoint = ""
1381                 cr.Container.Mounts = make(map[string]arvados.Mount)
1382                 cr.Container.Mounts = map[string]arvados.Mount{
1383                         "stdin": {Kind: "tmp"},
1384                 }
1385
1386                 _, err := cr.SetupMounts()
1387                 c.Check(err, NotNil)
1388                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1389                 os.RemoveAll(cr.ArvMountPoint)
1390                 cr.CleanupDirs()
1391                 checkEmpty()
1392         }
1393
1394         // git_tree mounts
1395         {
1396                 i = 0
1397                 cr.ArvMountPoint = ""
1398                 (*GitMountSuite)(nil).useTestGitServer(c)
1399                 cr.token = arvadostest.ActiveToken
1400                 cr.Container.Mounts = make(map[string]arvados.Mount)
1401                 cr.Container.Mounts = map[string]arvados.Mount{
1402                         "/tip": {
1403                                 Kind:   "git_tree",
1404                                 UUID:   arvadostest.Repository2UUID,
1405                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1406                                 Path:   "/",
1407                         },
1408                         "/non-tip": {
1409                                 Kind:   "git_tree",
1410                                 UUID:   arvadostest.Repository2UUID,
1411                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1412                                 Path:   "/",
1413                         },
1414                 }
1415                 cr.Container.OutputPath = "/tmp"
1416
1417                 bindmounts, err := cr.SetupMounts()
1418                 c.Check(err, IsNil)
1419
1420                 for path, mount := range bindmounts {
1421                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1422                 }
1423
1424                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1425                 c.Check(err, IsNil)
1426                 c.Check(string(data), Equals, "\000\001\002\003")
1427                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1428                 c.Check(err, FitsTypeOf, &os.PathError{})
1429                 c.Check(os.IsNotExist(err), Equals, true)
1430
1431                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1432                 c.Check(err, IsNil)
1433                 c.Check(string(data), Equals, "\000\001\002\003")
1434                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1435                 c.Check(err, IsNil)
1436                 c.Check(string(data), Equals, "testfile\n")
1437
1438                 cr.CleanupDirs()
1439                 checkEmpty()
1440         }
1441 }
1442
1443 func (s *TestSuite) TestStdout(c *C) {
1444         helperRecord := `{
1445                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1446                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1447                 "cwd": "/bin",
1448                 "environment": {"FROBIZ": "bilbo"},
1449                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1450                 "output_path": "/tmp",
1451                 "priority": 1,
1452                 "runtime_constraints": {},
1453                 "state": "Locked"
1454         }`
1455
1456         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1457                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1458         })
1459
1460         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1461         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1462         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1463 }
1464
1465 // Used by the TestStdoutWithWrongPath*()
1466 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1467         err := json.Unmarshal([]byte(record), &s.api.Container)
1468         c.Assert(err, IsNil)
1469         s.executor.runFunc = fn
1470         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1471         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1472                 return s.api, &KeepTestClient{}, nil, nil
1473         }
1474         return s.api, s.runner, s.runner.Run()
1475 }
1476
1477 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1478         _, _, err := s.stdoutErrorRunHelper(c, `{
1479     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1480     "output_path": "/tmp",
1481     "state": "Locked"
1482 }`, func() {})
1483         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1484 }
1485
1486 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1487         _, _, err := s.stdoutErrorRunHelper(c, `{
1488     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1489     "output_path": "/tmp",
1490     "state": "Locked"
1491 }`, func() {})
1492         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1493 }
1494
1495 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1496         _, _, err := s.stdoutErrorRunHelper(c, `{
1497     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1498     "output_path": "/tmp",
1499     "state": "Locked"
1500 }`, func() {})
1501         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1502 }
1503
1504 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1505         s.fullRunHelper(c, `{
1506     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1507     "container_image": "`+arvadostest.DockerImage112PDH+`",
1508     "cwd": "/bin",
1509     "environment": {},
1510     "mounts": {"/tmp": {"kind": "tmp"} },
1511     "output_path": "/tmp",
1512     "priority": 1,
1513     "runtime_constraints": {"API": true},
1514     "state": "Locked"
1515 }`, nil, 0, func() {
1516                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1517                 s.executor.exit <- 3
1518         })
1519         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1520         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1521 }
1522
1523 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1524         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1525         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1526         s.fullRunHelper(c, `{
1527     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1528     "container_image": "`+arvadostest.DockerImage112PDH+`",
1529     "cwd": "/bin",
1530     "environment": {},
1531     "mounts": {"/tmp": {"kind": "tmp"} },
1532     "output_path": "/tmp",
1533     "priority": 1,
1534     "runtime_constraints": {"API": true},
1535     "state": "Locked"
1536 }`, nil, 0, func() {
1537                 s.api.Container.Output = arvadostest.DockerImage112PDH
1538         })
1539
1540         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1541         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1542         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1543 }
1544
1545 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1546         helperRecord := `{
1547                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1548                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1549                 "cwd": "/bin",
1550                 "environment": {"FROBIZ": "bilbo"},
1551                 "mounts": {
1552         "/tmp": {"kind": "tmp"},
1553         "/tmp/foo": {"kind": "collection",
1554                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1555                      "exclude_from_output": true
1556         },
1557         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1558     },
1559                 "output_path": "/tmp",
1560                 "priority": 1,
1561                 "runtime_constraints": {},
1562                 "state": "Locked"
1563         }`
1564
1565         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1566
1567         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1568                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1569         })
1570
1571         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1572         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1573         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1574 }
1575
1576 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1577         helperRecord := `{
1578                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1579                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1580                 "cwd": "/bin",
1581                 "environment": {"FROBIZ": "bilbo"},
1582                 "mounts": {
1583         "/tmp": {"kind": "tmp"},
1584         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1585         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1586         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1587         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1588         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1589     },
1590                 "output_path": "/tmp",
1591                 "priority": 1,
1592                 "runtime_constraints": {},
1593                 "state": "Locked"
1594         }`
1595
1596         extraMounts := []string{
1597                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1598                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1599                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1600         }
1601
1602         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1603                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1604         })
1605
1606         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1607                 "/tmp":                   {realtemp + "/tmp1", false},
1608                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1609                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1610                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1611                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1612         })
1613
1614         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1615         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1616         for _, v := range api.Content {
1617                 if v["collection"] != nil {
1618                         c.Check(v["ensure_unique_name"], Equals, true)
1619                         collection := v["collection"].(arvadosclient.Dict)
1620                         if strings.Index(collection["name"].(string), "output") == 0 {
1621                                 manifest := collection["manifest_text"].(string)
1622
1623                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1624 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1625 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1626 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1627 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1628 `)
1629                         }
1630                 }
1631         }
1632 }
1633
1634 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1635         helperRecord := `{
1636                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1637                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1638                 "cwd": "/bin",
1639                 "environment": {"FROBIZ": "bilbo"},
1640                 "mounts": {
1641         "/tmp": {"kind": "tmp"},
1642         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1643         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1644     },
1645                 "output_path": "/tmp",
1646                 "priority": 1,
1647                 "runtime_constraints": {},
1648                 "state": "Locked"
1649         }`
1650
1651         extraMounts := []string{
1652                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1653         }
1654
1655         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1656                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1657         })
1658
1659         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1660         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1661         for _, v := range s.api.Content {
1662                 if v["collection"] != nil {
1663                         collection := v["collection"].(arvadosclient.Dict)
1664                         if strings.Index(collection["name"].(string), "output") == 0 {
1665                                 manifest := collection["manifest_text"].(string)
1666
1667                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1668 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1669 `)
1670                         }
1671                 }
1672         }
1673 }
1674
1675 func (s *TestSuite) TestOutputError(c *C) {
1676         helperRecord := `{
1677                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1678                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1679                 "cwd": "/bin",
1680                 "environment": {"FROBIZ": "bilbo"},
1681                 "mounts": {
1682                         "/tmp": {"kind": "tmp"}
1683                 },
1684                 "output_path": "/tmp",
1685                 "priority": 1,
1686                 "runtime_constraints": {},
1687                 "state": "Locked"
1688         }`
1689         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1690                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1691         })
1692
1693         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1694 }
1695
1696 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1697         helperRecord := `{
1698                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1699                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1700                 "cwd": "/bin",
1701                 "environment": {"FROBIZ": "bilbo"},
1702                 "mounts": {
1703         "/tmp": {"kind": "tmp"},
1704         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1705         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1706     },
1707                 "output_path": "/tmp",
1708                 "priority": 1,
1709                 "runtime_constraints": {},
1710                 "state": "Locked"
1711         }`
1712
1713         extraMounts := []string{
1714                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1715         }
1716
1717         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1718                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1719         })
1720
1721         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1722         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1723         for _, v := range api.Content {
1724                 if v["collection"] != nil {
1725                         collection := v["collection"].(arvadosclient.Dict)
1726                         if strings.Index(collection["name"].(string), "output") == 0 {
1727                                 manifest := collection["manifest_text"].(string)
1728                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1729 `)
1730                         }
1731                 }
1732         }
1733 }
1734
1735 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1736         helperRecord := `{
1737                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1738                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1739                 "cwd": "/bin",
1740                 "environment": {"FROBIZ": "bilbo"},
1741                 "mounts": {
1742         "/tmp": {"kind": "tmp"},
1743         "stdin": {"kind": "json", "content": "foo"},
1744         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1745     },
1746                 "output_path": "/tmp",
1747                 "priority": 1,
1748                 "runtime_constraints": {},
1749                 "state": "Locked"
1750         }`
1751
1752         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1753                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1754         })
1755
1756         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1757         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1758         for _, v := range api.Content {
1759                 if v["collection"] != nil {
1760                         collection := v["collection"].(arvadosclient.Dict)
1761                         if strings.Index(collection["name"].(string), "output") == 0 {
1762                                 manifest := collection["manifest_text"].(string)
1763                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1764 `)
1765                         }
1766                 }
1767         }
1768 }
1769
1770 func (s *TestSuite) TestStderrMount(c *C) {
1771         api, cr, _ := s.fullRunHelper(c, `{
1772     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1773     "container_image": "`+arvadostest.DockerImage112PDH+`",
1774     "cwd": ".",
1775     "environment": {},
1776     "mounts": {"/tmp": {"kind": "tmp"},
1777                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1778                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1779     "output_path": "/tmp",
1780     "priority": 1,
1781     "runtime_constraints": {},
1782     "state": "Locked"
1783 }`, nil, 1, func() {
1784                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1785                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1786         })
1787
1788         final := api.CalledWith("container.state", "Complete")
1789         c.Assert(final, NotNil)
1790         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1791         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1792
1793         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1794 }
1795
1796 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1797         s.api.callraw = true
1798         err := s.runner.fetchContainerRecord()
1799         c.Assert(err, IsNil)
1800         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1801         c.Logf("%#v", s.runner.Container)
1802         c.Check(err, IsNil)
1803         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1804 }
1805
1806 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1807         nextState := ""
1808         for _, setup := range []func(){
1809                 func() {
1810                         c.Log("// waitErr = ocl runtime error")
1811                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1812                         nextState = "Cancelled"
1813                 },
1814                 func() {
1815                         c.Log("// loadErr = cannot connect")
1816                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1817                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1818                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1819                         c.Assert(err, IsNil)
1820                         nextState = "Queued"
1821                 },
1822         } {
1823                 s.SetUpTest(c)
1824                 setup()
1825                 s.fullRunHelper(c, `{
1826     "command": ["echo", "hello world"],
1827     "container_image": "`+arvadostest.DockerImage112PDH+`",
1828     "cwd": ".",
1829     "environment": {},
1830     "mounts": {"/tmp": {"kind": "tmp"} },
1831     "output_path": "/tmp",
1832     "priority": 1,
1833     "runtime_constraints": {},
1834     "state": "Locked"
1835 }`, nil, 0, func() {})
1836                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1837                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1838                 if *brokenNodeHook != "" {
1839                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1840                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1841                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1842                 } else {
1843                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1844                 }
1845         }
1846 }
1847
1848 func (s *TestSuite) TestBadCommand(c *C) {
1849         for _, startError := range []string{
1850                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1851                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1852                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1853         } {
1854                 s.SetUpTest(c)
1855                 s.executor.startErr = errors.New(startError)
1856                 s.fullRunHelper(c, `{
1857     "command": ["echo", "hello world"],
1858     "container_image": "`+arvadostest.DockerImage112PDH+`",
1859     "cwd": ".",
1860     "environment": {},
1861     "mounts": {"/tmp": {"kind": "tmp"} },
1862     "output_path": "/tmp",
1863     "priority": 1,
1864     "runtime_constraints": {},
1865     "state": "Locked"
1866 }`, nil, 0, func() {})
1867                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1868                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1869         }
1870 }
1871
1872 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1873         helperRecord := `{
1874                 "command": ["true"],
1875                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1876                 "cwd": "/bin",
1877                 "mounts": {
1878                     "/tmp": {"kind": "tmp"},
1879                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1880                 },
1881                 "secret_mounts": {
1882                 },
1883                 "output_path": "/tmp",
1884                 "priority": 1,
1885                 "runtime_constraints": {},
1886                 "state": "Locked"
1887         }`
1888
1889         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1890                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1891                 c.Check(err, IsNil)
1892                 c.Check(string(content), Equals, "mypassword")
1893         })
1894
1895         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1896         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1897         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1898         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1899
1900         // under secret mounts, not captured in output
1901         helperRecord = `{
1902                 "command": ["true"],
1903                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1904                 "cwd": "/bin",
1905                 "mounts": {
1906                     "/tmp": {"kind": "tmp"}
1907                 },
1908                 "secret_mounts": {
1909                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1910                 },
1911                 "output_path": "/tmp",
1912                 "priority": 1,
1913                 "runtime_constraints": {},
1914                 "state": "Locked"
1915         }`
1916
1917         s.SetUpTest(c)
1918         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1919                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1920                 c.Check(err, IsNil)
1921                 c.Check(string(content), Equals, "mypassword")
1922         })
1923
1924         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1925         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1926         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1927         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1928 }
1929
1930 type FakeProcess struct {
1931         cmdLine []string
1932 }
1933
1934 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1935         return fp.cmdLine, nil
1936 }