lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 116         containerClient *arvados.Client) error {
 117         e.loaded = tarball
 118         return e.loadErr
 119 }
 120 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 121 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 122 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 123 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 124 func (e *stubExecutor) Close()                          { e.closed = true }
 125 func (e *stubExecutor) Wait(context.Context) (int, error) {
 126         return <-e.exit, e.waitErr
 127 }
 128
 129 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 130
 131 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 132 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 133 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 134
 135 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 136 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 137
 138 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 139 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 140 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 141 `
 142
 143 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 144
 145 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 146 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 147
 148 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 149 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 150
 151 func (client *ArvTestClient) Create(resourceType string,
 152         parameters arvadosclient.Dict,
 153         output interface{}) error {
 154
 155         client.Mutex.Lock()
 156         defer client.Mutex.Unlock()
 157
 158         client.Calls++
 159         client.Content = append(client.Content, parameters)
 160
 161         if resourceType == "logs" {
 162                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 163                 if client.Logs == nil {
 164                         client.Logs = make(map[string]*bytes.Buffer)
 165                 }
 166                 if client.Logs[et] == nil {
 167                         client.Logs[et] = &bytes.Buffer{}
 168                 }
 169                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 170         }
 171
 172         if resourceType == "collections" && output != nil {
 173                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 174                 outmap := output.(*arvados.Collection)
 175                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 176                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 177         }
 178
 179         return nil
 180 }
 181
 182 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 183         switch {
 184         case method == "GET" && resourceType == "containers" && action == "auth":
 185                 return json.Unmarshal([]byte(`{
 186                         "kind": "arvados#api_client_authorization",
 187                         "uuid": "`+fakeAuthUUID+`",
 188                         "api_token": "`+fakeAuthToken+`"
 189                         }`), output)
 190         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 191                 if client.secretMounts != nil {
 192                         return json.Unmarshal(client.secretMounts, output)
 193                 }
 194                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 195         default:
 196                 return fmt.Errorf("Not found")
 197         }
 198 }
 199
 200 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 201         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 202         var j []byte
 203         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 204                 j = []byte(`{
 205                         "kind": "arvados#nodeList",
 206                         "items": [{
 207                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 208                                 "hostname": "compute2",
 209                                 "properties": {"total_cpu_cores": 16}
 210                         }]}`)
 211         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 212                 if uuid == "" {
 213                         j, err = json.Marshal(map[string]interface{}{
 214                                 "items": []interface{}{client.Container},
 215                                 "kind":  "arvados#nodeList",
 216                         })
 217                 } else {
 218                         j, err = json.Marshal(client.Container)
 219                 }
 220         } else {
 221                 j = []byte(`{
 222                         "command": ["sleep", "1"],
 223                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 224                         "cwd": ".",
 225                         "environment": {},
 226                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 227                         "output_path": "/tmp",
 228                         "priority": 1,
 229                         "runtime_constraints": {}
 230                 }`)
 231         }
 232         return ioutil.NopCloser(bytes.NewReader(j)), err
 233 }
 234
 235 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 236         if resourceType == "collections" {
 237                 if uuid == hwPDH {
 238                         output.(*arvados.Collection).ManifestText = hwManifest
 239                 } else if uuid == otherPDH {
 240                         output.(*arvados.Collection).ManifestText = otherManifest
 241                 } else if uuid == normalizedWithSubdirsPDH {
 242                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 243                 } else if uuid == denormalizedWithSubdirsPDH {
 244                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 245                 }
 246         }
 247         if resourceType == "containers" {
 248                 (*output.(*arvados.Container)) = client.Container
 249         }
 250         return nil
 251 }
 252
 253 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 254         client.Mutex.Lock()
 255         defer client.Mutex.Unlock()
 256         client.Calls++
 257         client.Content = append(client.Content, parameters)
 258         if resourceType == "containers" {
 259                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 260                         client.WasSetRunning = true
 261                 }
 262         } else if resourceType == "collections" {
 263                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 264                 output.(*arvados.Collection).UUID = uuid
 265                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 266         }
 267         return nil
 268 }
 269
 270 var discoveryMap = map[string]interface{}{
 271         "defaultTrashLifetime":               float64(1209600),
 272         "crunchLimitLogBytesPerJob":          float64(67108864),
 273         "crunchLogThrottleBytes":             float64(65536),
 274         "crunchLogThrottlePeriod":            float64(60),
 275         "crunchLogThrottleLines":             float64(1024),
 276         "crunchLogPartialLineThrottlePeriod": float64(5),
 277         "crunchLogBytesPerEvent":             float64(4096),
 278         "crunchLogSecondsBetweenEvents":      float64(1),
 279 }
 280
 281 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 282         return discoveryMap[key], nil
 283 }
 284
 285 // CalledWith returns the parameters from the first API call whose
 286 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 287 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 288 // no call matches, it returns nil.
 289 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 290 call:
 291         for _, content := range client.Content {
 292                 var v interface{} = content
 293                 for _, k := range strings.Split(jpath, ".") {
 294                         if dict, ok := v.(arvadosclient.Dict); !ok {
 295                                 continue call
 296                         } else {
 297                                 v = dict[k]
 298                         }
 299                 }
 300                 if v == expect {
 301                         return content
 302                 }
 303         }
 304         return nil
 305 }
 306
 307 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 308         return locator, nil
 309 }
 310
 311 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 312         client.Content = opts.Data
 313         return arvados.BlockWriteResponse{
 314                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 315         }, nil
 316 }
 317
 318 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 319         return 0, errors.New("not implemented")
 320 }
 321
 322 func (client *KeepTestClient) ClearBlockCache() {
 323 }
 324
 325 func (client *KeepTestClient) Close() {
 326         client.Content = nil
 327 }
 328
 329 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 330         client.StorageClasses = sc
 331 }
 332
 333 type FileWrapper struct {
 334         io.ReadCloser
 335         len int64
 336 }
 337
 338 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 339         return nil, errors.New("not implemented")
 340 }
 341
 342 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 343         return 0, errors.New("not implemented")
 344 }
 345
 346 func (fw FileWrapper) Size() int64 {
 347         return fw.len
 348 }
 349
 350 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 351         return nil, errors.New("not implemented")
 352 }
 353
 354 func (fw FileWrapper) Truncate(int64) error {
 355         return errors.New("not implemented")
 356 }
 357
 358 func (fw FileWrapper) Write([]byte) (int, error) {
 359         return 0, errors.New("not implemented")
 360 }
 361
 362 func (fw FileWrapper) Sync() error {
 363         return errors.New("not implemented")
 364 }
 365
 366 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 367         if filename == hwImageID+".tar" {
 368                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 369                 client.Called = true
 370                 return FileWrapper{rdr, 1321984}, nil
 371         } else if filename == "/file1_in_main.txt" {
 372                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 373                 client.Called = true
 374                 return FileWrapper{rdr, 3}, nil
 375         }
 376         return nil, nil
 377 }
 378
 379 func (s *TestSuite) TestLoadImage(c *C) {
 380         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 381         s.runner.Container.Mounts = map[string]arvados.Mount{
 382                 "/out": {Kind: "tmp", Writable: true},
 383         }
 384         s.runner.Container.OutputPath = "/out"
 385
 386         _, err := s.runner.SetupMounts()
 387         c.Assert(err, IsNil)
 388
 389         imageID, err := s.runner.LoadImage()
 390         c.Check(err, IsNil)
 391         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 392         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 393
 394         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 395         s.executor.imageLoaded = false
 396         s.executor.loaded = ""
 397         s.executor.loadErr = errors.New("bork")
 398         imageID, err = s.runner.LoadImage()
 399         c.Check(err, ErrorMatches, ".*bork")
 400         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 401
 402         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 403         s.executor.imageLoaded = false
 404         s.executor.loaded = ""
 405         s.executor.loadErr = nil
 406         imageID, err = s.runner.LoadImage()
 407         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 408         c.Check(s.executor.loaded, Equals, "")
 409
 410         // if executor reports image is already loaded, LoadImage should not be called
 411         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 412         s.executor.imageLoaded = true
 413         s.executor.loaded = ""
 414         s.executor.loadErr = nil
 415         imageID, err = s.runner.LoadImage()
 416         c.Check(err, IsNil)
 417         c.Check(s.executor.loaded, Equals, "")
 418         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 419 }
 420
 421 type ArvErrorTestClient struct{}
 422
 423 func (ArvErrorTestClient) Create(resourceType string,
 424         parameters arvadosclient.Dict,
 425         output interface{}) error {
 426         return nil
 427 }
 428
 429 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 430         if method == "GET" && resourceType == "containers" && action == "auth" {
 431                 return nil
 432         }
 433         return errors.New("ArvError")
 434 }
 435
 436 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 437         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 438         return nil, errors.New("ArvError")
 439 }
 440
 441 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 442         return errors.New("ArvError")
 443 }
 444
 445 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 446         return nil
 447 }
 448
 449 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 450         return discoveryMap[key], nil
 451 }
 452
 453 type KeepErrorTestClient struct {
 454         KeepTestClient
 455 }
 456
 457 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 458         return nil, errors.New("KeepError")
 459 }
 460
 461 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 462         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 463 }
 464
 465 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 466         return "", errors.New("KeepError")
 467 }
 468
 469 type KeepReadErrorTestClient struct {
 470         KeepTestClient
 471 }
 472
 473 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 474         return 0, errors.New("KeepError")
 475 }
 476
 477 type ErrorReader struct {
 478         FileWrapper
 479 }
 480
 481 func (ErrorReader) Read(p []byte) (n int, err error) {
 482         return 0, errors.New("ErrorReader")
 483 }
 484
 485 func (ErrorReader) Seek(int64, int) (int64, error) {
 486         return 0, errors.New("ErrorReader")
 487 }
 488
 489 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 490         return ErrorReader{}, nil
 491 }
 492
 493 type ClosableBuffer struct {
 494         bytes.Buffer
 495 }
 496
 497 func (*ClosableBuffer) Close() error {
 498         return nil
 499 }
 500
 501 type TestLogs struct {
 502         Stdout ClosableBuffer
 503         Stderr ClosableBuffer
 504 }
 505
 506 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 507         if logstr == "stdout" {
 508                 return &tl.Stdout, nil
 509         }
 510         if logstr == "stderr" {
 511                 return &tl.Stderr, nil
 512         }
 513         return nil, errors.New("???")
 514 }
 515
 516 func dockerLog(fd byte, msg string) []byte {
 517         by := []byte(msg)
 518         header := make([]byte, 8+len(by))
 519         header[0] = fd
 520         header[7] = byte(len(by))
 521         copy(header[8:], by)
 522         return header
 523 }
 524
 525 func (s *TestSuite) TestRunContainer(c *C) {
 526         s.executor.runFunc = func() {
 527                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 528                 s.executor.exit <- 0
 529         }
 530
 531         var logs TestLogs
 532         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 533         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 534         s.runner.Container.Command = []string{"./hw"}
 535         s.runner.Container.OutputStorageClasses = []string{"default"}
 536
 537         imageID, err := s.runner.LoadImage()
 538         c.Assert(err, IsNil)
 539
 540         err = s.runner.CreateContainer(imageID, nil)
 541         c.Assert(err, IsNil)
 542
 543         err = s.runner.StartContainer()
 544         c.Assert(err, IsNil)
 545
 546         err = s.runner.WaitFinish()
 547         c.Assert(err, IsNil)
 548
 549         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 550         c.Check(logs.Stderr.String(), Equals, "")
 551 }
 552
 553 func (s *TestSuite) TestCommitLogs(c *C) {
 554         api := &ArvTestClient{}
 555         kc := &KeepTestClient{}
 556         defer kc.Close()
 557         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 558         c.Assert(err, IsNil)
 559         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 560
 561         cr.CrunchLog.Print("Hello world!")
 562         cr.CrunchLog.Print("Goodbye")
 563         cr.finalState = "Complete"
 564
 565         err = cr.CommitLogs()
 566         c.Check(err, IsNil)
 567
 568         c.Check(api.Calls, Equals, 2)
 569         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 570         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 571         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 572         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 573 }
 574
 575 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 576         api := &ArvTestClient{}
 577         kc := &KeepTestClient{}
 578         defer kc.Close()
 579         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 580         c.Assert(err, IsNil)
 581
 582         err = cr.UpdateContainerRunning()
 583         c.Check(err, IsNil)
 584
 585         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 586 }
 587
 588 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 589         api := &ArvTestClient{}
 590         kc := &KeepTestClient{}
 591         defer kc.Close()
 592         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 593         c.Assert(err, IsNil)
 594
 595         cr.LogsPDH = new(string)
 596         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 597
 598         cr.ExitCode = new(int)
 599         *cr.ExitCode = 42
 600         cr.finalState = "Complete"
 601
 602         err = cr.UpdateContainerFinal()
 603         c.Check(err, IsNil)
 604
 605         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 606         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 607         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 608 }
 609
 610 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 611         api := &ArvTestClient{}
 612         kc := &KeepTestClient{}
 613         defer kc.Close()
 614         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 615         c.Assert(err, IsNil)
 616         cr.cCancelled = true
 617         cr.finalState = "Cancelled"
 618
 619         err = cr.UpdateContainerFinal()
 620         c.Check(err, IsNil)
 621
 622         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 623         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 624         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 625 }
 626
 627 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 628 // dress rehearsal of the Run() function, starting from a JSON container record.
 629 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 630         err := json.Unmarshal([]byte(record), &s.api.Container)
 631         c.Assert(err, IsNil)
 632         initialState := s.api.Container.State
 633
 634         var sm struct {
 635                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 636         }
 637         err = json.Unmarshal([]byte(record), &sm)
 638         c.Check(err, IsNil)
 639         secretMounts, err := json.Marshal(sm)
 640         c.Assert(err, IsNil)
 641         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 642
 643         s.executor.runFunc = func() {
 644                 fn()
 645                 s.executor.exit <- exitCode
 646         }
 647
 648         s.runner.statInterval = 100 * time.Millisecond
 649         s.runner.containerWatchdogInterval = time.Second
 650         am := &ArvMountCmdLine{}
 651         s.runner.RunArvMount = am.ArvMountTest
 652
 653         realTemp := c.MkDir()
 654         tempcount := 0
 655         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 656                 tempcount++
 657                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 658                 err := os.Mkdir(d, os.ModePerm)
 659                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 660                         // Test case must have pre-populated the tempdir
 661                         err = nil
 662                 }
 663                 return d, err
 664         }
 665         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 666                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 667         }
 668
 669         if extraMounts != nil && len(extraMounts) > 0 {
 670                 err := s.runner.SetupArvMountPoint("keep")
 671                 c.Check(err, IsNil)
 672
 673                 for _, m := range extraMounts {
 674                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 675                 }
 676         }
 677
 678         err = s.runner.Run()
 679         if s.api.CalledWith("container.state", "Complete") != nil {
 680                 c.Check(err, IsNil)
 681         }
 682         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 683                 c.Check(s.api.WasSetRunning, Equals, true)
 684                 var lastupdate arvadosclient.Dict
 685                 for _, content := range s.api.Content {
 686                         if content["container"] != nil {
 687                                 lastupdate = content["container"].(arvadosclient.Dict)
 688                         }
 689                 }
 690                 if lastupdate["log"] == nil {
 691                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 692                 }
 693         }
 694
 695         if err != nil {
 696                 for k, v := range s.api.Logs {
 697                         c.Log(k)
 698                         c.Log(v.String())
 699                 }
 700         }
 701
 702         return s.api, s.runner, realTemp
 703 }
 704
 705 func (s *TestSuite) TestFullRunHello(c *C) {
 706         s.runner.enableMemoryLimit = true
 707         s.runner.networkMode = "default"
 708         s.fullRunHelper(c, `{
 709     "command": ["echo", "hello world"],
 710     "container_image": "`+arvadostest.DockerImage112PDH+`",
 711     "cwd": ".",
 712     "environment": {"foo":"bar","baz":"waz"},
 713     "mounts": {"/tmp": {"kind": "tmp"} },
 714     "output_path": "/tmp",
 715     "priority": 1,
 716     "runtime_constraints": {"vcpus":1,"ram":1000000},
 717     "state": "Locked",
 718     "output_storage_classes": ["default"]
 719 }`, nil, 0, func() {
 720                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 721                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 722                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 723                 c.Check(s.executor.created.VCPUs, Equals, 1)
 724                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 725                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 726                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 727                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 728         })
 729
 730         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 731         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 732         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 733         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 734         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 735 }
 736
 737 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 738         var ran bool
 739         s.fullRunHelper(c, `{
 740     "command": ["sleep", "3"],
 741     "container_image": "`+arvadostest.DockerImage112PDH+`",
 742     "cwd": ".",
 743     "environment": {},
 744     "mounts": {"/tmp": {"kind": "tmp"} },
 745     "output_path": "/tmp",
 746     "priority": 1,
 747     "runtime_constraints": {},
 748     "scheduling_parameters":{"max_run_time": 1},
 749     "state": "Running"
 750 }`, nil, 2, func() {
 751                 ran = true
 752         })
 753         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 754         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 755         c.Check(ran, Equals, false)
 756 }
 757
 758 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 759         s.fullRunHelper(c, `{
 760     "command": ["sleep", "3"],
 761     "container_image": "`+arvadostest.DockerImage112PDH+`",
 762     "cwd": ".",
 763     "environment": {},
 764     "mounts": {"/tmp": {"kind": "tmp"} },
 765     "output_path": "/tmp",
 766     "priority": 1,
 767     "runtime_constraints": {},
 768     "scheduling_parameters":{"max_run_time": 1},
 769     "state": "Locked"
 770 }`, nil, 0, func() {
 771                 time.Sleep(3 * time.Second)
 772         })
 773
 774         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 775         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 776 }
 777
 778 func (s *TestSuite) TestContainerWaitFails(c *C) {
 779         s.fullRunHelper(c, `{
 780     "command": ["sleep", "3"],
 781     "container_image": "`+arvadostest.DockerImage112PDH+`",
 782     "cwd": ".",
 783     "mounts": {"/tmp": {"kind": "tmp"} },
 784     "output_path": "/tmp",
 785     "priority": 1,
 786     "state": "Locked"
 787 }`, nil, 0, func() {
 788                 s.executor.waitErr = errors.New("Container is not running")
 789         })
 790
 791         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 792         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 793 }
 794
 795 func (s *TestSuite) TestCrunchstat(c *C) {
 796         s.fullRunHelper(c, `{
 797                 "command": ["sleep", "1"],
 798                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 799                 "cwd": ".",
 800                 "environment": {},
 801                 "mounts": {"/tmp": {"kind": "tmp"} },
 802                 "output_path": "/tmp",
 803                 "priority": 1,
 804                 "runtime_constraints": {},
 805                 "state": "Locked"
 806         }`, nil, 0, func() {
 807                 time.Sleep(time.Second)
 808         })
 809
 810         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 811         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 812
 813         // We didn't actually start a container, so crunchstat didn't
 814         // find accounting files and therefore didn't log any stats.
 815         // It should have logged a "can't find accounting files"
 816         // message after one poll interval, though, so we can confirm
 817         // it's alive:
 818         c.Assert(s.api.Logs["crunchstat"], NotNil)
 819         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 820
 821         // The "files never appeared" log assures us that we called
 822         // (*crunchstat.Reporter)Stop(), and that we set it up with
 823         // the correct container ID "abcde":
 824         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 825 }
 826
 827 func (s *TestSuite) TestNodeInfoLog(c *C) {
 828         os.Setenv("SLURMD_NODENAME", "compute2")
 829         s.fullRunHelper(c, `{
 830                 "command": ["sleep", "1"],
 831                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 832                 "cwd": ".",
 833                 "environment": {},
 834                 "mounts": {"/tmp": {"kind": "tmp"} },
 835                 "output_path": "/tmp",
 836                 "priority": 1,
 837                 "runtime_constraints": {},
 838                 "state": "Locked"
 839         }`, nil, 0,
 840                 func() {
 841                         time.Sleep(time.Second)
 842                 })
 843
 844         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 845         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 846
 847         c.Assert(s.api.Logs["node"], NotNil)
 848         json := s.api.Logs["node"].String()
 849         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 850         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 851         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 852
 853         c.Assert(s.api.Logs["node-info"], NotNil)
 854         json = s.api.Logs["node-info"].String()
 855         c.Check(json, Matches, `(?ms).*Host Information.*`)
 856         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 857         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 858         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 859         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 860 }
 861
 862 func (s *TestSuite) TestContainerRecordLog(c *C) {
 863         s.fullRunHelper(c, `{
 864                 "command": ["sleep", "1"],
 865                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 866                 "cwd": ".",
 867                 "environment": {},
 868                 "mounts": {"/tmp": {"kind": "tmp"} },
 869                 "output_path": "/tmp",
 870                 "priority": 1,
 871                 "runtime_constraints": {},
 872                 "state": "Locked"
 873         }`, nil, 0,
 874                 func() {
 875                         time.Sleep(time.Second)
 876                 })
 877
 878         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 879         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 880
 881         c.Assert(s.api.Logs["container"], NotNil)
 882         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 883 }
 884
 885 func (s *TestSuite) TestFullRunStderr(c *C) {
 886         s.fullRunHelper(c, `{
 887     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 888     "container_image": "`+arvadostest.DockerImage112PDH+`",
 889     "cwd": ".",
 890     "environment": {},
 891     "mounts": {"/tmp": {"kind": "tmp"} },
 892     "output_path": "/tmp",
 893     "priority": 1,
 894     "runtime_constraints": {},
 895     "state": "Locked"
 896 }`, nil, 1, func() {
 897                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 898                 fmt.Fprintln(s.executor.created.Stderr, "world")
 899         })
 900
 901         final := s.api.CalledWith("container.state", "Complete")
 902         c.Assert(final, NotNil)
 903         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 904         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 905
 906         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 907         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 908 }
 909
 910 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 911         s.fullRunHelper(c, `{
 912     "command": ["pwd"],
 913     "container_image": "`+arvadostest.DockerImage112PDH+`",
 914     "cwd": ".",
 915     "environment": {},
 916     "mounts": {"/tmp": {"kind": "tmp"} },
 917     "output_path": "/tmp",
 918     "priority": 1,
 919     "runtime_constraints": {},
 920     "state": "Locked"
 921 }`, nil, 0, func() {
 922                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 923         })
 924
 925         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 926         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 927         c.Log(s.api.Logs["stdout"])
 928         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 929 }
 930
 931 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 932         s.fullRunHelper(c, `{
 933     "command": ["pwd"],
 934     "container_image": "`+arvadostest.DockerImage112PDH+`",
 935     "cwd": "/bin",
 936     "environment": {},
 937     "mounts": {"/tmp": {"kind": "tmp"} },
 938     "output_path": "/tmp",
 939     "priority": 1,
 940     "runtime_constraints": {},
 941     "state": "Locked"
 942 }`, nil, 0, func() {
 943                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 944         })
 945
 946         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 947         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 948         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 949 }
 950
 951 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 952         s.fullRunHelper(c, `{
 953     "command": ["pwd"],
 954     "container_image": "`+arvadostest.DockerImage112PDH+`",
 955     "cwd": "/bin",
 956     "environment": {},
 957     "mounts": {"/tmp": {"kind": "tmp"} },
 958     "output_path": "/tmp",
 959     "priority": 1,
 960     "runtime_constraints": {},
 961     "state": "Locked",
 962     "output_storage_classes": ["foo", "bar"]
 963 }`, nil, 0, func() {
 964                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 965         })
 966
 967         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 968         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 969         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 970         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 971         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 972 }
 973
 974 func (s *TestSuite) TestStopOnSignal(c *C) {
 975         s.executor.runFunc = func() {
 976                 s.executor.created.Stdout.Write([]byte("foo\n"))
 977                 s.runner.SigChan <- syscall.SIGINT
 978         }
 979         s.testStopContainer(c)
 980 }
 981
 982 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 983         s.executor.runFunc = func() {
 984                 s.executor.created.Stdout.Write([]byte("foo\n"))
 985                 s.runner.ArvMountExit <- nil
 986                 close(s.runner.ArvMountExit)
 987         }
 988         s.runner.ArvMountExit = make(chan error)
 989         s.testStopContainer(c)
 990 }
 991
 992 func (s *TestSuite) testStopContainer(c *C) {
 993         record := `{
 994     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 995     "container_image": "` + arvadostest.DockerImage112PDH + `",
 996     "cwd": ".",
 997     "environment": {},
 998     "mounts": {"/tmp": {"kind": "tmp"} },
 999     "output_path": "/tmp",
1000     "priority": 1,
1001     "runtime_constraints": {},
1002     "state": "Locked"
1003 }`
1004
1005         err := json.Unmarshal([]byte(record), &s.api.Container)
1006         c.Assert(err, IsNil)
1007
1008         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1009         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1010                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1011         }
1012
1013         done := make(chan error)
1014         go func() {
1015                 done <- s.runner.Run()
1016         }()
1017         select {
1018         case <-time.After(20 * time.Second):
1019                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1020                 c.Fatal("timed out")
1021         case err = <-done:
1022                 c.Check(err, IsNil)
1023         }
1024         for k, v := range s.api.Logs {
1025                 c.Log(k)
1026                 c.Log(v.String(), "\n")
1027         }
1028
1029         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1030         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1031         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1032 }
1033
1034 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1035         s.fullRunHelper(c, `{
1036     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1037     "container_image": "`+arvadostest.DockerImage112PDH+`",
1038     "cwd": "/bin",
1039     "environment": {"FROBIZ": "bilbo"},
1040     "mounts": {"/tmp": {"kind": "tmp"} },
1041     "output_path": "/tmp",
1042     "priority": 1,
1043     "runtime_constraints": {},
1044     "state": "Locked"
1045 }`, nil, 0, func() {
1046                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1047         })
1048
1049         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1050         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1051         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1052 }
1053
1054 type ArvMountCmdLine struct {
1055         Cmd   []string
1056         token string
1057 }
1058
1059 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1060         am.Cmd = c
1061         am.token = token
1062         return nil, nil
1063 }
1064
1065 func stubCert(temp string) string {
1066         path := temp + "/ca-certificates.crt"
1067         crt, _ := os.Create(path)
1068         crt.Close()
1069         arvadosclient.CertFiles = []string{path}
1070         return path
1071 }
1072
1073 func (s *TestSuite) TestSetupMounts(c *C) {
1074         cr := s.runner
1075         am := &ArvMountCmdLine{}
1076         cr.RunArvMount = am.ArvMountTest
1077         cr.ContainerArvClient = &ArvTestClient{}
1078         cr.ContainerKeepClient = &KeepTestClient{}
1079         cr.Container.OutputStorageClasses = []string{"default"}
1080
1081         realTemp := c.MkDir()
1082         certTemp := c.MkDir()
1083         stubCertPath := stubCert(certTemp)
1084         cr.parentTemp = realTemp
1085
1086         i := 0
1087         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1088                 i++
1089                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1090                 err := os.Mkdir(d, os.ModePerm)
1091                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1092                         // Test case must have pre-populated the tempdir
1093                         err = nil
1094                 }
1095                 return d, err
1096         }
1097
1098         checkEmpty := func() {
1099                 // Should be deleted.
1100                 _, err := os.Stat(realTemp)
1101                 c.Assert(os.IsNotExist(err), Equals, true)
1102
1103                 // Now recreate it for the next test.
1104                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1105         }
1106
1107         {
1108                 i = 0
1109                 cr.ArvMountPoint = ""
1110                 cr.Container.Mounts = make(map[string]arvados.Mount)
1111                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1112                 cr.Container.OutputPath = "/tmp"
1113                 cr.statInterval = 5 * time.Second
1114                 bindmounts, err := cr.SetupMounts()
1115                 c.Check(err, IsNil)
1116                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1117                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1118                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1119                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1120                 os.RemoveAll(cr.ArvMountPoint)
1121                 cr.CleanupDirs()
1122                 checkEmpty()
1123         }
1124
1125         {
1126                 i = 0
1127                 cr.ArvMountPoint = ""
1128                 cr.Container.Mounts = make(map[string]arvados.Mount)
1129                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1130                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1131                 cr.Container.OutputPath = "/out"
1132                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1133
1134                 bindmounts, err := cr.SetupMounts()
1135                 c.Check(err, IsNil)
1136                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1137                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1138                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1139                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1140                 os.RemoveAll(cr.ArvMountPoint)
1141                 cr.CleanupDirs()
1142                 checkEmpty()
1143         }
1144
1145         {
1146                 i = 0
1147                 cr.ArvMountPoint = ""
1148                 cr.Container.Mounts = make(map[string]arvados.Mount)
1149                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1150                 cr.Container.OutputPath = "/tmp"
1151                 cr.Container.RuntimeConstraints.API = true
1152                 cr.Container.OutputStorageClasses = []string{"default"}
1153
1154                 bindmounts, err := cr.SetupMounts()
1155                 c.Check(err, IsNil)
1156                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1157                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1158                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1159                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1160                 os.RemoveAll(cr.ArvMountPoint)
1161                 cr.CleanupDirs()
1162                 checkEmpty()
1163
1164                 cr.Container.RuntimeConstraints.API = false
1165         }
1166
1167         {
1168                 i = 0
1169                 cr.ArvMountPoint = ""
1170                 cr.Container.Mounts = map[string]arvados.Mount{
1171                         "/keeptmp": {Kind: "collection", Writable: true},
1172                 }
1173                 cr.Container.OutputPath = "/keeptmp"
1174
1175                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1176
1177                 bindmounts, err := cr.SetupMounts()
1178                 c.Check(err, IsNil)
1179                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1180                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1181                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1182                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1183                 os.RemoveAll(cr.ArvMountPoint)
1184                 cr.CleanupDirs()
1185                 checkEmpty()
1186         }
1187
1188         {
1189                 i = 0
1190                 cr.ArvMountPoint = ""
1191                 cr.Container.Mounts = map[string]arvados.Mount{
1192                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1193                         "/keepout": {Kind: "collection", Writable: true},
1194                 }
1195                 cr.Container.OutputPath = "/keepout"
1196
1197                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1198                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1199
1200                 bindmounts, err := cr.SetupMounts()
1201                 c.Check(err, IsNil)
1202                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1203                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1204                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1205                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1206                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1207                         "/keepout": {realTemp + "/keep1/tmp0", false},
1208                 })
1209                 os.RemoveAll(cr.ArvMountPoint)
1210                 cr.CleanupDirs()
1211                 checkEmpty()
1212         }
1213
1214         {
1215                 i = 0
1216                 cr.ArvMountPoint = ""
1217                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1218                 cr.Container.Mounts = map[string]arvados.Mount{
1219                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1220                         "/keepout": {Kind: "collection", Writable: true},
1221                 }
1222                 cr.Container.OutputPath = "/keepout"
1223
1224                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1225                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1226
1227                 bindmounts, err := cr.SetupMounts()
1228                 c.Check(err, IsNil)
1229                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1230                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1231                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1232                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1233                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1234                         "/keepout": {realTemp + "/keep1/tmp0", false},
1235                 })
1236                 os.RemoveAll(cr.ArvMountPoint)
1237                 cr.CleanupDirs()
1238                 checkEmpty()
1239         }
1240
1241         for _, test := range []struct {
1242                 in  interface{}
1243                 out string
1244         }{
1245                 {in: "foo", out: `"foo"`},
1246                 {in: nil, out: `null`},
1247                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1248         } {
1249                 i = 0
1250                 cr.ArvMountPoint = ""
1251                 cr.Container.Mounts = map[string]arvados.Mount{
1252                         "/mnt/test.json": {Kind: "json", Content: test.in},
1253                 }
1254                 bindmounts, err := cr.SetupMounts()
1255                 c.Check(err, IsNil)
1256                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1257                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1258                 })
1259                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1260                 c.Check(err, IsNil)
1261                 c.Check(content, DeepEquals, []byte(test.out))
1262                 os.RemoveAll(cr.ArvMountPoint)
1263                 cr.CleanupDirs()
1264                 checkEmpty()
1265         }
1266
1267         for _, test := range []struct {
1268                 in  interface{}
1269                 out string
1270         }{
1271                 {in: "foo", out: `foo`},
1272                 {in: nil, out: "error"},
1273                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1274         } {
1275                 i = 0
1276                 cr.ArvMountPoint = ""
1277                 cr.Container.Mounts = map[string]arvados.Mount{
1278                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1279                 }
1280                 bindmounts, err := cr.SetupMounts()
1281                 if test.out == "error" {
1282                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1283                 } else {
1284                         c.Check(err, IsNil)
1285                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1286                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1287                         })
1288                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1289                         c.Check(err, IsNil)
1290                         c.Check(content, DeepEquals, []byte(test.out))
1291                 }
1292                 os.RemoveAll(cr.ArvMountPoint)
1293                 cr.CleanupDirs()
1294                 checkEmpty()
1295         }
1296
1297         // Read-only mount points are allowed underneath output_dir mount point
1298         {
1299                 i = 0
1300                 cr.ArvMountPoint = ""
1301                 cr.Container.Mounts = make(map[string]arvados.Mount)
1302                 cr.Container.Mounts = map[string]arvados.Mount{
1303                         "/tmp":     {Kind: "tmp"},
1304                         "/tmp/foo": {Kind: "collection"},
1305                 }
1306                 cr.Container.OutputPath = "/tmp"
1307
1308                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1309
1310                 bindmounts, err := cr.SetupMounts()
1311                 c.Check(err, IsNil)
1312                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1313                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1314                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1315                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1316                         "/tmp":     {realTemp + "/tmp2", false},
1317                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1318                 })
1319                 os.RemoveAll(cr.ArvMountPoint)
1320                 cr.CleanupDirs()
1321                 checkEmpty()
1322         }
1323
1324         // Writable mount points copied to output_dir mount point
1325         {
1326                 i = 0
1327                 cr.ArvMountPoint = ""
1328                 cr.Container.Mounts = make(map[string]arvados.Mount)
1329                 cr.Container.Mounts = map[string]arvados.Mount{
1330                         "/tmp": {Kind: "tmp"},
1331                         "/tmp/foo": {Kind: "collection",
1332                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1333                                 Writable:         true},
1334                         "/tmp/bar": {Kind: "collection",
1335                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1336                                 Path:             "baz",
1337                                 Writable:         true},
1338                 }
1339                 cr.Container.OutputPath = "/tmp"
1340
1341                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1342                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1343
1344                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1345                 rf.Write([]byte("bar"))
1346                 rf.Close()
1347
1348                 _, err := cr.SetupMounts()
1349                 c.Check(err, IsNil)
1350                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1351                 c.Check(err, IsNil)
1352                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1353                 c.Check(err, IsNil)
1354                 os.RemoveAll(cr.ArvMountPoint)
1355                 cr.CleanupDirs()
1356                 checkEmpty()
1357         }
1358
1359         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1360         {
1361                 i = 0
1362                 cr.ArvMountPoint = ""
1363                 cr.Container.Mounts = make(map[string]arvados.Mount)
1364                 cr.Container.Mounts = map[string]arvados.Mount{
1365                         "/tmp":     {Kind: "tmp"},
1366                         "/tmp/foo": {Kind: "tmp"},
1367                 }
1368                 cr.Container.OutputPath = "/tmp"
1369
1370                 _, err := cr.SetupMounts()
1371                 c.Check(err, NotNil)
1372                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1373                 os.RemoveAll(cr.ArvMountPoint)
1374                 cr.CleanupDirs()
1375                 checkEmpty()
1376         }
1377
1378         // Only mount point of kind 'collection' is allowed for stdin
1379         {
1380                 i = 0
1381                 cr.ArvMountPoint = ""
1382                 cr.Container.Mounts = make(map[string]arvados.Mount)
1383                 cr.Container.Mounts = map[string]arvados.Mount{
1384                         "stdin": {Kind: "tmp"},
1385                 }
1386
1387                 _, err := cr.SetupMounts()
1388                 c.Check(err, NotNil)
1389                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1390                 os.RemoveAll(cr.ArvMountPoint)
1391                 cr.CleanupDirs()
1392                 checkEmpty()
1393         }
1394
1395         // git_tree mounts
1396         {
1397                 i = 0
1398                 cr.ArvMountPoint = ""
1399                 (*GitMountSuite)(nil).useTestGitServer(c)
1400                 cr.token = arvadostest.ActiveToken
1401                 cr.Container.Mounts = make(map[string]arvados.Mount)
1402                 cr.Container.Mounts = map[string]arvados.Mount{
1403                         "/tip": {
1404                                 Kind:   "git_tree",
1405                                 UUID:   arvadostest.Repository2UUID,
1406                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1407                                 Path:   "/",
1408                         },
1409                         "/non-tip": {
1410                                 Kind:   "git_tree",
1411                                 UUID:   arvadostest.Repository2UUID,
1412                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1413                                 Path:   "/",
1414                         },
1415                 }
1416                 cr.Container.OutputPath = "/tmp"
1417
1418                 bindmounts, err := cr.SetupMounts()
1419                 c.Check(err, IsNil)
1420
1421                 for path, mount := range bindmounts {
1422                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1423                 }
1424
1425                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1426                 c.Check(err, IsNil)
1427                 c.Check(string(data), Equals, "\000\001\002\003")
1428                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1429                 c.Check(err, FitsTypeOf, &os.PathError{})
1430                 c.Check(os.IsNotExist(err), Equals, true)
1431
1432                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1433                 c.Check(err, IsNil)
1434                 c.Check(string(data), Equals, "\000\001\002\003")
1435                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1436                 c.Check(err, IsNil)
1437                 c.Check(string(data), Equals, "testfile\n")
1438
1439                 cr.CleanupDirs()
1440                 checkEmpty()
1441         }
1442 }
1443
1444 func (s *TestSuite) TestStdout(c *C) {
1445         helperRecord := `{
1446                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1447                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1448                 "cwd": "/bin",
1449                 "environment": {"FROBIZ": "bilbo"},
1450                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1451                 "output_path": "/tmp",
1452                 "priority": 1,
1453                 "runtime_constraints": {},
1454                 "state": "Locked"
1455         }`
1456
1457         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1458                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1459         })
1460
1461         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1462         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1463         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1464 }
1465
1466 // Used by the TestStdoutWithWrongPath*()
1467 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1468         err := json.Unmarshal([]byte(record), &s.api.Container)
1469         c.Assert(err, IsNil)
1470         s.executor.runFunc = fn
1471         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1472         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1473                 return s.api, &KeepTestClient{}, nil, nil
1474         }
1475         return s.api, s.runner, s.runner.Run()
1476 }
1477
1478 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1479         _, _, err := s.stdoutErrorRunHelper(c, `{
1480     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1481     "output_path": "/tmp",
1482     "state": "Locked"
1483 }`, func() {})
1484         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1485 }
1486
1487 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1488         _, _, err := s.stdoutErrorRunHelper(c, `{
1489     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1490     "output_path": "/tmp",
1491     "state": "Locked"
1492 }`, func() {})
1493         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1494 }
1495
1496 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1497         _, _, err := s.stdoutErrorRunHelper(c, `{
1498     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1499     "output_path": "/tmp",
1500     "state": "Locked"
1501 }`, func() {})
1502         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1503 }
1504
1505 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1506         s.fullRunHelper(c, `{
1507     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1508     "container_image": "`+arvadostest.DockerImage112PDH+`",
1509     "cwd": "/bin",
1510     "environment": {},
1511     "mounts": {"/tmp": {"kind": "tmp"} },
1512     "output_path": "/tmp",
1513     "priority": 1,
1514     "runtime_constraints": {"API": true},
1515     "state": "Locked"
1516 }`, nil, 0, func() {
1517                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1518                 s.executor.exit <- 3
1519         })
1520         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1521         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1522 }
1523
1524 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1525         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1526         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1527         s.fullRunHelper(c, `{
1528     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1529     "container_image": "`+arvadostest.DockerImage112PDH+`",
1530     "cwd": "/bin",
1531     "environment": {},
1532     "mounts": {"/tmp": {"kind": "tmp"} },
1533     "output_path": "/tmp",
1534     "priority": 1,
1535     "runtime_constraints": {"API": true},
1536     "state": "Locked"
1537 }`, nil, 0, func() {
1538                 s.api.Container.Output = arvadostest.DockerImage112PDH
1539         })
1540
1541         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1542         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1543         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1544 }
1545
1546 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1547         helperRecord := `{
1548                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1549                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1550                 "cwd": "/bin",
1551                 "environment": {"FROBIZ": "bilbo"},
1552                 "mounts": {
1553         "/tmp": {"kind": "tmp"},
1554         "/tmp/foo": {"kind": "collection",
1555                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1556                      "exclude_from_output": true
1557         },
1558         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1559     },
1560                 "output_path": "/tmp",
1561                 "priority": 1,
1562                 "runtime_constraints": {},
1563                 "state": "Locked"
1564         }`
1565
1566         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1567
1568         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1569                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1570         })
1571
1572         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1573         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1574         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1575 }
1576
1577 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1578         helperRecord := `{
1579                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1580                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1581                 "cwd": "/bin",
1582                 "environment": {"FROBIZ": "bilbo"},
1583                 "mounts": {
1584         "/tmp": {"kind": "tmp"},
1585         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1586         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1587         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1588         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1589         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1590     },
1591                 "output_path": "/tmp",
1592                 "priority": 1,
1593                 "runtime_constraints": {},
1594                 "state": "Locked"
1595         }`
1596
1597         extraMounts := []string{
1598                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1599                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1600                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1601         }
1602
1603         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1604                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1605         })
1606
1607         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1608                 "/tmp":                   {realtemp + "/tmp1", false},
1609                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1610                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1611                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1612                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1613         })
1614
1615         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1616         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1617         for _, v := range api.Content {
1618                 if v["collection"] != nil {
1619                         c.Check(v["ensure_unique_name"], Equals, true)
1620                         collection := v["collection"].(arvadosclient.Dict)
1621                         if strings.Index(collection["name"].(string), "output") == 0 {
1622                                 manifest := collection["manifest_text"].(string)
1623
1624                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1625 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1626 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1627 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1628 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1629 `)
1630                         }
1631                 }
1632         }
1633 }
1634
1635 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1636         helperRecord := `{
1637                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1638                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1639                 "cwd": "/bin",
1640                 "environment": {"FROBIZ": "bilbo"},
1641                 "mounts": {
1642         "/tmp": {"kind": "tmp"},
1643         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1644         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1645     },
1646                 "output_path": "/tmp",
1647                 "priority": 1,
1648                 "runtime_constraints": {},
1649                 "state": "Locked"
1650         }`
1651
1652         extraMounts := []string{
1653                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1654         }
1655
1656         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1657                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1658         })
1659
1660         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1661         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1662         for _, v := range s.api.Content {
1663                 if v["collection"] != nil {
1664                         collection := v["collection"].(arvadosclient.Dict)
1665                         if strings.Index(collection["name"].(string), "output") == 0 {
1666                                 manifest := collection["manifest_text"].(string)
1667
1668                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1669 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1670 `)
1671                         }
1672                 }
1673         }
1674 }
1675
1676 func (s *TestSuite) TestOutputError(c *C) {
1677         helperRecord := `{
1678                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1679                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1680                 "cwd": "/bin",
1681                 "environment": {"FROBIZ": "bilbo"},
1682                 "mounts": {
1683                         "/tmp": {"kind": "tmp"}
1684                 },
1685                 "output_path": "/tmp",
1686                 "priority": 1,
1687                 "runtime_constraints": {},
1688                 "state": "Locked"
1689         }`
1690         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1691                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1692         })
1693
1694         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1695 }
1696
1697 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1698         helperRecord := `{
1699                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1700                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1701                 "cwd": "/bin",
1702                 "environment": {"FROBIZ": "bilbo"},
1703                 "mounts": {
1704         "/tmp": {"kind": "tmp"},
1705         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1706         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1707     },
1708                 "output_path": "/tmp",
1709                 "priority": 1,
1710                 "runtime_constraints": {},
1711                 "state": "Locked"
1712         }`
1713
1714         extraMounts := []string{
1715                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1716         }
1717
1718         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1719                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1720         })
1721
1722         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1723         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1724         for _, v := range api.Content {
1725                 if v["collection"] != nil {
1726                         collection := v["collection"].(arvadosclient.Dict)
1727                         if strings.Index(collection["name"].(string), "output") == 0 {
1728                                 manifest := collection["manifest_text"].(string)
1729                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1730 `)
1731                         }
1732                 }
1733         }
1734 }
1735
1736 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1737         helperRecord := `{
1738                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1739                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1740                 "cwd": "/bin",
1741                 "environment": {"FROBIZ": "bilbo"},
1742                 "mounts": {
1743         "/tmp": {"kind": "tmp"},
1744         "stdin": {"kind": "json", "content": "foo"},
1745         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1746     },
1747                 "output_path": "/tmp",
1748                 "priority": 1,
1749                 "runtime_constraints": {},
1750                 "state": "Locked"
1751         }`
1752
1753         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1754                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1755         })
1756
1757         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1758         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1759         for _, v := range api.Content {
1760                 if v["collection"] != nil {
1761                         collection := v["collection"].(arvadosclient.Dict)
1762                         if strings.Index(collection["name"].(string), "output") == 0 {
1763                                 manifest := collection["manifest_text"].(string)
1764                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1765 `)
1766                         }
1767                 }
1768         }
1769 }
1770
1771 func (s *TestSuite) TestStderrMount(c *C) {
1772         api, cr, _ := s.fullRunHelper(c, `{
1773     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1774     "container_image": "`+arvadostest.DockerImage112PDH+`",
1775     "cwd": ".",
1776     "environment": {},
1777     "mounts": {"/tmp": {"kind": "tmp"},
1778                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1779                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1780     "output_path": "/tmp",
1781     "priority": 1,
1782     "runtime_constraints": {},
1783     "state": "Locked"
1784 }`, nil, 1, func() {
1785                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1786                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1787         })
1788
1789         final := api.CalledWith("container.state", "Complete")
1790         c.Assert(final, NotNil)
1791         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1792         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1793
1794         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1795 }
1796
1797 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1798         s.api.callraw = true
1799         err := s.runner.fetchContainerRecord()
1800         c.Assert(err, IsNil)
1801         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1802         c.Logf("%#v", s.runner.Container)
1803         c.Check(err, IsNil)
1804         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1805 }
1806
1807 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1808         nextState := ""
1809         for _, setup := range []func(){
1810                 func() {
1811                         c.Log("// waitErr = ocl runtime error")
1812                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1813                         nextState = "Cancelled"
1814                 },
1815                 func() {
1816                         c.Log("// loadErr = cannot connect")
1817                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1818                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1819                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1820                         c.Assert(err, IsNil)
1821                         nextState = "Queued"
1822                 },
1823         } {
1824                 s.SetUpTest(c)
1825                 setup()
1826                 s.fullRunHelper(c, `{
1827     "command": ["echo", "hello world"],
1828     "container_image": "`+arvadostest.DockerImage112PDH+`",
1829     "cwd": ".",
1830     "environment": {},
1831     "mounts": {"/tmp": {"kind": "tmp"} },
1832     "output_path": "/tmp",
1833     "priority": 1,
1834     "runtime_constraints": {},
1835     "state": "Locked"
1836 }`, nil, 0, func() {})
1837                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1838                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1839                 if *brokenNodeHook != "" {
1840                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1841                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1842                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1843                 } else {
1844                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1845                 }
1846         }
1847 }
1848
1849 func (s *TestSuite) TestBadCommand(c *C) {
1850         for _, startError := range []string{
1851                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1852                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1853                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1854         } {
1855                 s.SetUpTest(c)
1856                 s.executor.startErr = errors.New(startError)
1857                 s.fullRunHelper(c, `{
1858     "command": ["echo", "hello world"],
1859     "container_image": "`+arvadostest.DockerImage112PDH+`",
1860     "cwd": ".",
1861     "environment": {},
1862     "mounts": {"/tmp": {"kind": "tmp"} },
1863     "output_path": "/tmp",
1864     "priority": 1,
1865     "runtime_constraints": {},
1866     "state": "Locked"
1867 }`, nil, 0, func() {})
1868                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1869                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1870         }
1871 }
1872
1873 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1874         helperRecord := `{
1875                 "command": ["true"],
1876                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1877                 "cwd": "/bin",
1878                 "mounts": {
1879                     "/tmp": {"kind": "tmp"},
1880                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1881                 },
1882                 "secret_mounts": {
1883                 },
1884                 "output_path": "/tmp",
1885                 "priority": 1,
1886                 "runtime_constraints": {},
1887                 "state": "Locked"
1888         }`
1889
1890         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1891                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1892                 c.Check(err, IsNil)
1893                 c.Check(string(content), Equals, "mypassword")
1894         })
1895
1896         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1897         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1898         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1899         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1900
1901         // under secret mounts, not captured in output
1902         helperRecord = `{
1903                 "command": ["true"],
1904                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1905                 "cwd": "/bin",
1906                 "mounts": {
1907                     "/tmp": {"kind": "tmp"}
1908                 },
1909                 "secret_mounts": {
1910                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1911                 },
1912                 "output_path": "/tmp",
1913                 "priority": 1,
1914                 "runtime_constraints": {},
1915                 "state": "Locked"
1916         }`
1917
1918         s.SetUpTest(c)
1919         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1920                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1921                 c.Check(err, IsNil)
1922                 c.Check(string(content), Equals, "mypassword")
1923         })
1924
1925         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1926         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1927         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1928         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1929 }
1930
1931 type FakeProcess struct {
1932         cmdLine []string
1933 }
1934
1935 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1936         return fp.cmdLine, nil
1937 }