lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) ImageLoaded(imageID string) bool { return e.imageLoaded }
 116 func (e *stubExecutor) LoadImage(filename string) error { e.loaded = filename; return e.loadErr }
 117 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 118 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 119 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 120 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 121 func (e *stubExecutor) Close()                          { e.closed = true }
 122 func (e *stubExecutor) Wait(context.Context) (int, error) {
 123         return <-e.exit, e.waitErr
 124 }
 125
 126 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 127
 128 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 129 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 130 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 131
 132 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 133 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 134
 135 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 136 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 137 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 138 `
 139
 140 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 141
 142 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 143 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 144
 145 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 146 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 147
 148 func (client *ArvTestClient) Create(resourceType string,
 149         parameters arvadosclient.Dict,
 150         output interface{}) error {
 151
 152         client.Mutex.Lock()
 153         defer client.Mutex.Unlock()
 154
 155         client.Calls++
 156         client.Content = append(client.Content, parameters)
 157
 158         if resourceType == "logs" {
 159                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 160                 if client.Logs == nil {
 161                         client.Logs = make(map[string]*bytes.Buffer)
 162                 }
 163                 if client.Logs[et] == nil {
 164                         client.Logs[et] = &bytes.Buffer{}
 165                 }
 166                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 167         }
 168
 169         if resourceType == "collections" && output != nil {
 170                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 171                 outmap := output.(*arvados.Collection)
 172                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 173                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 174         }
 175
 176         return nil
 177 }
 178
 179 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 180         switch {
 181         case method == "GET" && resourceType == "containers" && action == "auth":
 182                 return json.Unmarshal([]byte(`{
 183                         "kind": "arvados#api_client_authorization",
 184                         "uuid": "`+fakeAuthUUID+`",
 185                         "api_token": "`+fakeAuthToken+`"
 186                         }`), output)
 187         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 188                 if client.secretMounts != nil {
 189                         return json.Unmarshal(client.secretMounts, output)
 190                 }
 191                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 192         default:
 193                 return fmt.Errorf("Not found")
 194         }
 195 }
 196
 197 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 198         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 199         var j []byte
 200         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 201                 j = []byte(`{
 202                         "kind": "arvados#nodeList",
 203                         "items": [{
 204                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 205                                 "hostname": "compute2",
 206                                 "properties": {"total_cpu_cores": 16}
 207                         }]}`)
 208         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 209                 if uuid == "" {
 210                         j, err = json.Marshal(map[string]interface{}{
 211                                 "items": []interface{}{client.Container},
 212                                 "kind":  "arvados#nodeList",
 213                         })
 214                 } else {
 215                         j, err = json.Marshal(client.Container)
 216                 }
 217         } else {
 218                 j = []byte(`{
 219                         "command": ["sleep", "1"],
 220                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 221                         "cwd": ".",
 222                         "environment": {},
 223                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 224                         "output_path": "/tmp",
 225                         "priority": 1,
 226                         "runtime_constraints": {}
 227                 }`)
 228         }
 229         return ioutil.NopCloser(bytes.NewReader(j)), err
 230 }
 231
 232 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 233         if resourceType == "collections" {
 234                 if uuid == hwPDH {
 235                         output.(*arvados.Collection).ManifestText = hwManifest
 236                 } else if uuid == otherPDH {
 237                         output.(*arvados.Collection).ManifestText = otherManifest
 238                 } else if uuid == normalizedWithSubdirsPDH {
 239                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 240                 } else if uuid == denormalizedWithSubdirsPDH {
 241                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 242                 }
 243         }
 244         if resourceType == "containers" {
 245                 (*output.(*arvados.Container)) = client.Container
 246         }
 247         return nil
 248 }
 249
 250 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 251         client.Mutex.Lock()
 252         defer client.Mutex.Unlock()
 253         client.Calls++
 254         client.Content = append(client.Content, parameters)
 255         if resourceType == "containers" {
 256                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 257                         client.WasSetRunning = true
 258                 }
 259         } else if resourceType == "collections" {
 260                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 261                 output.(*arvados.Collection).UUID = uuid
 262                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 263         }
 264         return nil
 265 }
 266
 267 var discoveryMap = map[string]interface{}{
 268         "defaultTrashLifetime":               float64(1209600),
 269         "crunchLimitLogBytesPerJob":          float64(67108864),
 270         "crunchLogThrottleBytes":             float64(65536),
 271         "crunchLogThrottlePeriod":            float64(60),
 272         "crunchLogThrottleLines":             float64(1024),
 273         "crunchLogPartialLineThrottlePeriod": float64(5),
 274         "crunchLogBytesPerEvent":             float64(4096),
 275         "crunchLogSecondsBetweenEvents":      float64(1),
 276 }
 277
 278 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 279         return discoveryMap[key], nil
 280 }
 281
 282 // CalledWith returns the parameters from the first API call whose
 283 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 284 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 285 // no call matches, it returns nil.
 286 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 287 call:
 288         for _, content := range client.Content {
 289                 var v interface{} = content
 290                 for _, k := range strings.Split(jpath, ".") {
 291                         if dict, ok := v.(arvadosclient.Dict); !ok {
 292                                 continue call
 293                         } else {
 294                                 v = dict[k]
 295                         }
 296                 }
 297                 if v == expect {
 298                         return content
 299                 }
 300         }
 301         return nil
 302 }
 303
 304 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 305         return locator, nil
 306 }
 307
 308 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 309         client.Content = opts.Data
 310         return arvados.BlockWriteResponse{
 311                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 312         }, nil
 313 }
 314
 315 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 316         return 0, errors.New("not implemented")
 317 }
 318
 319 func (client *KeepTestClient) ClearBlockCache() {
 320 }
 321
 322 func (client *KeepTestClient) Close() {
 323         client.Content = nil
 324 }
 325
 326 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 327         client.StorageClasses = sc
 328 }
 329
 330 type FileWrapper struct {
 331         io.ReadCloser
 332         len int64
 333 }
 334
 335 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 336         return nil, errors.New("not implemented")
 337 }
 338
 339 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 340         return 0, errors.New("not implemented")
 341 }
 342
 343 func (fw FileWrapper) Size() int64 {
 344         return fw.len
 345 }
 346
 347 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 348         return nil, errors.New("not implemented")
 349 }
 350
 351 func (fw FileWrapper) Truncate(int64) error {
 352         return errors.New("not implemented")
 353 }
 354
 355 func (fw FileWrapper) Write([]byte) (int, error) {
 356         return 0, errors.New("not implemented")
 357 }
 358
 359 func (fw FileWrapper) Sync() error {
 360         return errors.New("not implemented")
 361 }
 362
 363 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 364         if filename == hwImageID+".tar" {
 365                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 366                 client.Called = true
 367                 return FileWrapper{rdr, 1321984}, nil
 368         } else if filename == "/file1_in_main.txt" {
 369                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 370                 client.Called = true
 371                 return FileWrapper{rdr, 3}, nil
 372         }
 373         return nil, nil
 374 }
 375
 376 func (s *TestSuite) TestLoadImage(c *C) {
 377         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 378         s.runner.Container.Mounts = map[string]arvados.Mount{
 379                 "/out": {Kind: "tmp", Writable: true},
 380         }
 381         s.runner.Container.OutputPath = "/out"
 382
 383         _, err := s.runner.SetupMounts()
 384         c.Assert(err, IsNil)
 385
 386         imageID, err := s.runner.LoadImage()
 387         c.Check(err, IsNil)
 388         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 389         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 390
 391         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 392         s.executor.imageLoaded = false
 393         s.executor.loaded = ""
 394         s.executor.loadErr = errors.New("bork")
 395         imageID, err = s.runner.LoadImage()
 396         c.Check(err, ErrorMatches, ".*bork")
 397         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 398
 399         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 400         s.executor.imageLoaded = false
 401         s.executor.loaded = ""
 402         s.executor.loadErr = nil
 403         imageID, err = s.runner.LoadImage()
 404         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 405         c.Check(s.executor.loaded, Equals, "")
 406
 407         // if executor reports image is already loaded, LoadImage should not be called
 408         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 409         s.executor.imageLoaded = true
 410         s.executor.loaded = ""
 411         s.executor.loadErr = nil
 412         imageID, err = s.runner.LoadImage()
 413         c.Check(err, IsNil)
 414         c.Check(s.executor.loaded, Equals, "")
 415         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 416 }
 417
 418 type ArvErrorTestClient struct{}
 419
 420 func (ArvErrorTestClient) Create(resourceType string,
 421         parameters arvadosclient.Dict,
 422         output interface{}) error {
 423         return nil
 424 }
 425
 426 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 427         if method == "GET" && resourceType == "containers" && action == "auth" {
 428                 return nil
 429         }
 430         return errors.New("ArvError")
 431 }
 432
 433 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 434         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 435         return nil, errors.New("ArvError")
 436 }
 437
 438 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 439         return errors.New("ArvError")
 440 }
 441
 442 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 443         return nil
 444 }
 445
 446 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 447         return discoveryMap[key], nil
 448 }
 449
 450 type KeepErrorTestClient struct {
 451         KeepTestClient
 452 }
 453
 454 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 455         return nil, errors.New("KeepError")
 456 }
 457
 458 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 459         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 460 }
 461
 462 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 463         return "", errors.New("KeepError")
 464 }
 465
 466 type KeepReadErrorTestClient struct {
 467         KeepTestClient
 468 }
 469
 470 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 471         return 0, errors.New("KeepError")
 472 }
 473
 474 type ErrorReader struct {
 475         FileWrapper
 476 }
 477
 478 func (ErrorReader) Read(p []byte) (n int, err error) {
 479         return 0, errors.New("ErrorReader")
 480 }
 481
 482 func (ErrorReader) Seek(int64, int) (int64, error) {
 483         return 0, errors.New("ErrorReader")
 484 }
 485
 486 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 487         return ErrorReader{}, nil
 488 }
 489
 490 type ClosableBuffer struct {
 491         bytes.Buffer
 492 }
 493
 494 func (*ClosableBuffer) Close() error {
 495         return nil
 496 }
 497
 498 type TestLogs struct {
 499         Stdout ClosableBuffer
 500         Stderr ClosableBuffer
 501 }
 502
 503 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 504         if logstr == "stdout" {
 505                 return &tl.Stdout, nil
 506         }
 507         if logstr == "stderr" {
 508                 return &tl.Stderr, nil
 509         }
 510         return nil, errors.New("???")
 511 }
 512
 513 func dockerLog(fd byte, msg string) []byte {
 514         by := []byte(msg)
 515         header := make([]byte, 8+len(by))
 516         header[0] = fd
 517         header[7] = byte(len(by))
 518         copy(header[8:], by)
 519         return header
 520 }
 521
 522 func (s *TestSuite) TestRunContainer(c *C) {
 523         s.executor.runFunc = func() {
 524                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 525                 s.executor.exit <- 0
 526         }
 527
 528         var logs TestLogs
 529         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 530         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 531         s.runner.Container.Command = []string{"./hw"}
 532         s.runner.Container.OutputStorageClasses = []string{"default"}
 533
 534         imageID, err := s.runner.LoadImage()
 535         c.Assert(err, IsNil)
 536
 537         err = s.runner.CreateContainer(imageID, nil)
 538         c.Assert(err, IsNil)
 539
 540         err = s.runner.StartContainer()
 541         c.Assert(err, IsNil)
 542
 543         err = s.runner.WaitFinish()
 544         c.Assert(err, IsNil)
 545
 546         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 547         c.Check(logs.Stderr.String(), Equals, "")
 548 }
 549
 550 func (s *TestSuite) TestCommitLogs(c *C) {
 551         api := &ArvTestClient{}
 552         kc := &KeepTestClient{}
 553         defer kc.Close()
 554         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 555         c.Assert(err, IsNil)
 556         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 557
 558         cr.CrunchLog.Print("Hello world!")
 559         cr.CrunchLog.Print("Goodbye")
 560         cr.finalState = "Complete"
 561
 562         err = cr.CommitLogs()
 563         c.Check(err, IsNil)
 564
 565         c.Check(api.Calls, Equals, 2)
 566         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 567         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 568         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 569         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 570 }
 571
 572 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 573         api := &ArvTestClient{}
 574         kc := &KeepTestClient{}
 575         defer kc.Close()
 576         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 577         c.Assert(err, IsNil)
 578
 579         err = cr.UpdateContainerRunning()
 580         c.Check(err, IsNil)
 581
 582         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 583 }
 584
 585 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 586         api := &ArvTestClient{}
 587         kc := &KeepTestClient{}
 588         defer kc.Close()
 589         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 590         c.Assert(err, IsNil)
 591
 592         cr.LogsPDH = new(string)
 593         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 594
 595         cr.ExitCode = new(int)
 596         *cr.ExitCode = 42
 597         cr.finalState = "Complete"
 598
 599         err = cr.UpdateContainerFinal()
 600         c.Check(err, IsNil)
 601
 602         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 603         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 604         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 605 }
 606
 607 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 608         api := &ArvTestClient{}
 609         kc := &KeepTestClient{}
 610         defer kc.Close()
 611         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 612         c.Assert(err, IsNil)
 613         cr.cCancelled = true
 614         cr.finalState = "Cancelled"
 615
 616         err = cr.UpdateContainerFinal()
 617         c.Check(err, IsNil)
 618
 619         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 620         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 621         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 622 }
 623
 624 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 625 // dress rehearsal of the Run() function, starting from a JSON container record.
 626 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 627         err := json.Unmarshal([]byte(record), &s.api.Container)
 628         c.Assert(err, IsNil)
 629         initialState := s.api.Container.State
 630
 631         var sm struct {
 632                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 633         }
 634         err = json.Unmarshal([]byte(record), &sm)
 635         c.Check(err, IsNil)
 636         secretMounts, err := json.Marshal(sm)
 637         c.Assert(err, IsNil)
 638         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 639
 640         s.executor.runFunc = func() {
 641                 fn()
 642                 s.executor.exit <- exitCode
 643         }
 644
 645         s.runner.statInterval = 100 * time.Millisecond
 646         s.runner.containerWatchdogInterval = time.Second
 647         am := &ArvMountCmdLine{}
 648         s.runner.RunArvMount = am.ArvMountTest
 649
 650         realTemp := c.MkDir()
 651         tempcount := 0
 652         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 653                 tempcount++
 654                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 655                 err := os.Mkdir(d, os.ModePerm)
 656                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 657                         // Test case must have pre-populated the tempdir
 658                         err = nil
 659                 }
 660                 return d, err
 661         }
 662         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 663                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 664         }
 665
 666         if extraMounts != nil && len(extraMounts) > 0 {
 667                 err := s.runner.SetupArvMountPoint("keep")
 668                 c.Check(err, IsNil)
 669
 670                 for _, m := range extraMounts {
 671                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 672                 }
 673         }
 674
 675         err = s.runner.Run()
 676         if s.api.CalledWith("container.state", "Complete") != nil {
 677                 c.Check(err, IsNil)
 678         }
 679         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 680                 c.Check(s.api.WasSetRunning, Equals, true)
 681                 var lastupdate arvadosclient.Dict
 682                 for _, content := range s.api.Content {
 683                         if content["container"] != nil {
 684                                 lastupdate = content["container"].(arvadosclient.Dict)
 685                         }
 686                 }
 687                 if lastupdate["log"] == nil {
 688                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 689                 }
 690         }
 691
 692         if err != nil {
 693                 for k, v := range s.api.Logs {
 694                         c.Log(k)
 695                         c.Log(v.String())
 696                 }
 697         }
 698
 699         return s.api, s.runner, realTemp
 700 }
 701
 702 func (s *TestSuite) TestFullRunHello(c *C) {
 703         s.runner.enableMemoryLimit = true
 704         s.runner.networkMode = "default"
 705         s.fullRunHelper(c, `{
 706     "command": ["echo", "hello world"],
 707     "container_image": "`+arvadostest.DockerImage112PDH+`",
 708     "cwd": ".",
 709     "environment": {"foo":"bar","baz":"waz"},
 710     "mounts": {"/tmp": {"kind": "tmp"} },
 711     "output_path": "/tmp",
 712     "priority": 1,
 713     "runtime_constraints": {"vcpus":1,"ram":1000000},
 714     "state": "Locked",
 715     "output_storage_classes": ["default"]
 716 }`, nil, 0, func() {
 717                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 718                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 719                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 720                 c.Check(s.executor.created.VCPUs, Equals, 1)
 721                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 722                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 723                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 724                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 725         })
 726
 727         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 728         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 729         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 730         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 731         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 732 }
 733
 734 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 735         var ran bool
 736         s.fullRunHelper(c, `{
 737     "command": ["sleep", "3"],
 738     "container_image": "`+arvadostest.DockerImage112PDH+`",
 739     "cwd": ".",
 740     "environment": {},
 741     "mounts": {"/tmp": {"kind": "tmp"} },
 742     "output_path": "/tmp",
 743     "priority": 1,
 744     "runtime_constraints": {},
 745     "scheduling_parameters":{"max_run_time": 1},
 746     "state": "Running"
 747 }`, nil, 2, func() {
 748                 ran = true
 749         })
 750         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 751         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 752         c.Check(ran, Equals, false)
 753 }
 754
 755 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 756         s.fullRunHelper(c, `{
 757     "command": ["sleep", "3"],
 758     "container_image": "`+arvadostest.DockerImage112PDH+`",
 759     "cwd": ".",
 760     "environment": {},
 761     "mounts": {"/tmp": {"kind": "tmp"} },
 762     "output_path": "/tmp",
 763     "priority": 1,
 764     "runtime_constraints": {},
 765     "scheduling_parameters":{"max_run_time": 1},
 766     "state": "Locked"
 767 }`, nil, 0, func() {
 768                 time.Sleep(3 * time.Second)
 769         })
 770
 771         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 772         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 773 }
 774
 775 func (s *TestSuite) TestContainerWaitFails(c *C) {
 776         s.fullRunHelper(c, `{
 777     "command": ["sleep", "3"],
 778     "container_image": "`+arvadostest.DockerImage112PDH+`",
 779     "cwd": ".",
 780     "mounts": {"/tmp": {"kind": "tmp"} },
 781     "output_path": "/tmp",
 782     "priority": 1,
 783     "state": "Locked"
 784 }`, nil, 0, func() {
 785                 s.executor.waitErr = errors.New("Container is not running")
 786         })
 787
 788         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 789         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 790 }
 791
 792 func (s *TestSuite) TestCrunchstat(c *C) {
 793         s.fullRunHelper(c, `{
 794                 "command": ["sleep", "1"],
 795                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 796                 "cwd": ".",
 797                 "environment": {},
 798                 "mounts": {"/tmp": {"kind": "tmp"} },
 799                 "output_path": "/tmp",
 800                 "priority": 1,
 801                 "runtime_constraints": {},
 802                 "state": "Locked"
 803         }`, nil, 0, func() {
 804                 time.Sleep(time.Second)
 805         })
 806
 807         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 808         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 809
 810         // We didn't actually start a container, so crunchstat didn't
 811         // find accounting files and therefore didn't log any stats.
 812         // It should have logged a "can't find accounting files"
 813         // message after one poll interval, though, so we can confirm
 814         // it's alive:
 815         c.Assert(s.api.Logs["crunchstat"], NotNil)
 816         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 817
 818         // The "files never appeared" log assures us that we called
 819         // (*crunchstat.Reporter)Stop(), and that we set it up with
 820         // the correct container ID "abcde":
 821         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 822 }
 823
 824 func (s *TestSuite) TestNodeInfoLog(c *C) {
 825         os.Setenv("SLURMD_NODENAME", "compute2")
 826         s.fullRunHelper(c, `{
 827                 "command": ["sleep", "1"],
 828                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 829                 "cwd": ".",
 830                 "environment": {},
 831                 "mounts": {"/tmp": {"kind": "tmp"} },
 832                 "output_path": "/tmp",
 833                 "priority": 1,
 834                 "runtime_constraints": {},
 835                 "state": "Locked"
 836         }`, nil, 0,
 837                 func() {
 838                         time.Sleep(time.Second)
 839                 })
 840
 841         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 842         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 843
 844         c.Assert(s.api.Logs["node"], NotNil)
 845         json := s.api.Logs["node"].String()
 846         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 847         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 848         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 849
 850         c.Assert(s.api.Logs["node-info"], NotNil)
 851         json = s.api.Logs["node-info"].String()
 852         c.Check(json, Matches, `(?ms).*Host Information.*`)
 853         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 854         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 855         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 856         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 857 }
 858
 859 func (s *TestSuite) TestContainerRecordLog(c *C) {
 860         s.fullRunHelper(c, `{
 861                 "command": ["sleep", "1"],
 862                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 863                 "cwd": ".",
 864                 "environment": {},
 865                 "mounts": {"/tmp": {"kind": "tmp"} },
 866                 "output_path": "/tmp",
 867                 "priority": 1,
 868                 "runtime_constraints": {},
 869                 "state": "Locked"
 870         }`, nil, 0,
 871                 func() {
 872                         time.Sleep(time.Second)
 873                 })
 874
 875         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 876         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 877
 878         c.Assert(s.api.Logs["container"], NotNil)
 879         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 880 }
 881
 882 func (s *TestSuite) TestFullRunStderr(c *C) {
 883         s.fullRunHelper(c, `{
 884     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 885     "container_image": "`+arvadostest.DockerImage112PDH+`",
 886     "cwd": ".",
 887     "environment": {},
 888     "mounts": {"/tmp": {"kind": "tmp"} },
 889     "output_path": "/tmp",
 890     "priority": 1,
 891     "runtime_constraints": {},
 892     "state": "Locked"
 893 }`, nil, 1, func() {
 894                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 895                 fmt.Fprintln(s.executor.created.Stderr, "world")
 896         })
 897
 898         final := s.api.CalledWith("container.state", "Complete")
 899         c.Assert(final, NotNil)
 900         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 901         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 902
 903         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 904         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 905 }
 906
 907 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 908         s.fullRunHelper(c, `{
 909     "command": ["pwd"],
 910     "container_image": "`+arvadostest.DockerImage112PDH+`",
 911     "cwd": ".",
 912     "environment": {},
 913     "mounts": {"/tmp": {"kind": "tmp"} },
 914     "output_path": "/tmp",
 915     "priority": 1,
 916     "runtime_constraints": {},
 917     "state": "Locked"
 918 }`, nil, 0, func() {
 919                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 920         })
 921
 922         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 923         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 924         c.Log(s.api.Logs["stdout"])
 925         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 926 }
 927
 928 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 929         s.fullRunHelper(c, `{
 930     "command": ["pwd"],
 931     "container_image": "`+arvadostest.DockerImage112PDH+`",
 932     "cwd": "/bin",
 933     "environment": {},
 934     "mounts": {"/tmp": {"kind": "tmp"} },
 935     "output_path": "/tmp",
 936     "priority": 1,
 937     "runtime_constraints": {},
 938     "state": "Locked"
 939 }`, nil, 0, func() {
 940                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 941         })
 942
 943         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 944         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 945         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 946 }
 947
 948 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 949         s.fullRunHelper(c, `{
 950     "command": ["pwd"],
 951     "container_image": "`+arvadostest.DockerImage112PDH+`",
 952     "cwd": "/bin",
 953     "environment": {},
 954     "mounts": {"/tmp": {"kind": "tmp"} },
 955     "output_path": "/tmp",
 956     "priority": 1,
 957     "runtime_constraints": {},
 958     "state": "Locked",
 959     "output_storage_classes": ["foo", "bar"]
 960 }`, nil, 0, func() {
 961                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 962         })
 963
 964         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 965         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 966         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 967         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 968         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 969 }
 970
 971 func (s *TestSuite) TestStopOnSignal(c *C) {
 972         s.executor.runFunc = func() {
 973                 s.executor.created.Stdout.Write([]byte("foo\n"))
 974                 s.runner.SigChan <- syscall.SIGINT
 975         }
 976         s.testStopContainer(c)
 977 }
 978
 979 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 980         s.executor.runFunc = func() {
 981                 s.executor.created.Stdout.Write([]byte("foo\n"))
 982                 s.runner.ArvMountExit <- nil
 983                 close(s.runner.ArvMountExit)
 984         }
 985         s.runner.ArvMountExit = make(chan error)
 986         s.testStopContainer(c)
 987 }
 988
 989 func (s *TestSuite) testStopContainer(c *C) {
 990         record := `{
 991     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 992     "container_image": "` + arvadostest.DockerImage112PDH + `",
 993     "cwd": ".",
 994     "environment": {},
 995     "mounts": {"/tmp": {"kind": "tmp"} },
 996     "output_path": "/tmp",
 997     "priority": 1,
 998     "runtime_constraints": {},
 999     "state": "Locked"
1000 }`
1001
1002         err := json.Unmarshal([]byte(record), &s.api.Container)
1003         c.Assert(err, IsNil)
1004
1005         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1006         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1007                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1008         }
1009
1010         done := make(chan error)
1011         go func() {
1012                 done <- s.runner.Run()
1013         }()
1014         select {
1015         case <-time.After(20 * time.Second):
1016                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1017                 c.Fatal("timed out")
1018         case err = <-done:
1019                 c.Check(err, IsNil)
1020         }
1021         for k, v := range s.api.Logs {
1022                 c.Log(k)
1023                 c.Log(v.String(), "\n")
1024         }
1025
1026         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1027         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1028         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1029 }
1030
1031 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1032         s.fullRunHelper(c, `{
1033     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1034     "container_image": "`+arvadostest.DockerImage112PDH+`",
1035     "cwd": "/bin",
1036     "environment": {"FROBIZ": "bilbo"},
1037     "mounts": {"/tmp": {"kind": "tmp"} },
1038     "output_path": "/tmp",
1039     "priority": 1,
1040     "runtime_constraints": {},
1041     "state": "Locked"
1042 }`, nil, 0, func() {
1043                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1044         })
1045
1046         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1047         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1048         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1049 }
1050
1051 type ArvMountCmdLine struct {
1052         Cmd   []string
1053         token string
1054 }
1055
1056 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1057         am.Cmd = c
1058         am.token = token
1059         return nil, nil
1060 }
1061
1062 func stubCert(temp string) string {
1063         path := temp + "/ca-certificates.crt"
1064         crt, _ := os.Create(path)
1065         crt.Close()
1066         arvadosclient.CertFiles = []string{path}
1067         return path
1068 }
1069
1070 func (s *TestSuite) TestSetupMounts(c *C) {
1071         cr := s.runner
1072         am := &ArvMountCmdLine{}
1073         cr.RunArvMount = am.ArvMountTest
1074         cr.ContainerArvClient = &ArvTestClient{}
1075         cr.ContainerKeepClient = &KeepTestClient{}
1076         cr.Container.OutputStorageClasses = []string{"default"}
1077
1078         realTemp := c.MkDir()
1079         certTemp := c.MkDir()
1080         stubCertPath := stubCert(certTemp)
1081         cr.parentTemp = realTemp
1082
1083         i := 0
1084         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1085                 i++
1086                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1087                 err := os.Mkdir(d, os.ModePerm)
1088                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1089                         // Test case must have pre-populated the tempdir
1090                         err = nil
1091                 }
1092                 return d, err
1093         }
1094
1095         checkEmpty := func() {
1096                 // Should be deleted.
1097                 _, err := os.Stat(realTemp)
1098                 c.Assert(os.IsNotExist(err), Equals, true)
1099
1100                 // Now recreate it for the next test.
1101                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1102         }
1103
1104         {
1105                 i = 0
1106                 cr.ArvMountPoint = ""
1107                 cr.Container.Mounts = make(map[string]arvados.Mount)
1108                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1109                 cr.Container.OutputPath = "/tmp"
1110                 cr.statInterval = 5 * time.Second
1111                 bindmounts, err := cr.SetupMounts()
1112                 c.Check(err, IsNil)
1113                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1114                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1115                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1116                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1117                 os.RemoveAll(cr.ArvMountPoint)
1118                 cr.CleanupDirs()
1119                 checkEmpty()
1120         }
1121
1122         {
1123                 i = 0
1124                 cr.ArvMountPoint = ""
1125                 cr.Container.Mounts = make(map[string]arvados.Mount)
1126                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1127                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1128                 cr.Container.OutputPath = "/out"
1129                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1130
1131                 bindmounts, err := cr.SetupMounts()
1132                 c.Check(err, IsNil)
1133                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1134                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1135                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1136                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1137                 os.RemoveAll(cr.ArvMountPoint)
1138                 cr.CleanupDirs()
1139                 checkEmpty()
1140         }
1141
1142         {
1143                 i = 0
1144                 cr.ArvMountPoint = ""
1145                 cr.Container.Mounts = make(map[string]arvados.Mount)
1146                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1147                 cr.Container.OutputPath = "/tmp"
1148                 cr.Container.RuntimeConstraints.API = true
1149                 cr.Container.OutputStorageClasses = []string{"default"}
1150
1151                 bindmounts, err := cr.SetupMounts()
1152                 c.Check(err, IsNil)
1153                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1154                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1155                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1156                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1157                 os.RemoveAll(cr.ArvMountPoint)
1158                 cr.CleanupDirs()
1159                 checkEmpty()
1160
1161                 cr.Container.RuntimeConstraints.API = false
1162         }
1163
1164         {
1165                 i = 0
1166                 cr.ArvMountPoint = ""
1167                 cr.Container.Mounts = map[string]arvados.Mount{
1168                         "/keeptmp": {Kind: "collection", Writable: true},
1169                 }
1170                 cr.Container.OutputPath = "/keeptmp"
1171
1172                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1173
1174                 bindmounts, err := cr.SetupMounts()
1175                 c.Check(err, IsNil)
1176                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1177                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1178                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1179                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1180                 os.RemoveAll(cr.ArvMountPoint)
1181                 cr.CleanupDirs()
1182                 checkEmpty()
1183         }
1184
1185         {
1186                 i = 0
1187                 cr.ArvMountPoint = ""
1188                 cr.Container.Mounts = map[string]arvados.Mount{
1189                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1190                         "/keepout": {Kind: "collection", Writable: true},
1191                 }
1192                 cr.Container.OutputPath = "/keepout"
1193
1194                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1195                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1196
1197                 bindmounts, err := cr.SetupMounts()
1198                 c.Check(err, IsNil)
1199                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1200                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1201                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1202                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1203                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1204                         "/keepout": {realTemp + "/keep1/tmp0", false},
1205                 })
1206                 os.RemoveAll(cr.ArvMountPoint)
1207                 cr.CleanupDirs()
1208                 checkEmpty()
1209         }
1210
1211         {
1212                 i = 0
1213                 cr.ArvMountPoint = ""
1214                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1215                 cr.Container.Mounts = map[string]arvados.Mount{
1216                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1217                         "/keepout": {Kind: "collection", Writable: true},
1218                 }
1219                 cr.Container.OutputPath = "/keepout"
1220
1221                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1222                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1223
1224                 bindmounts, err := cr.SetupMounts()
1225                 c.Check(err, IsNil)
1226                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1227                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1228                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1229                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1230                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1231                         "/keepout": {realTemp + "/keep1/tmp0", false},
1232                 })
1233                 os.RemoveAll(cr.ArvMountPoint)
1234                 cr.CleanupDirs()
1235                 checkEmpty()
1236         }
1237
1238         for _, test := range []struct {
1239                 in  interface{}
1240                 out string
1241         }{
1242                 {in: "foo", out: `"foo"`},
1243                 {in: nil, out: `null`},
1244                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1245         } {
1246                 i = 0
1247                 cr.ArvMountPoint = ""
1248                 cr.Container.Mounts = map[string]arvados.Mount{
1249                         "/mnt/test.json": {Kind: "json", Content: test.in},
1250                 }
1251                 bindmounts, err := cr.SetupMounts()
1252                 c.Check(err, IsNil)
1253                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1254                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1255                 })
1256                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1257                 c.Check(err, IsNil)
1258                 c.Check(content, DeepEquals, []byte(test.out))
1259                 os.RemoveAll(cr.ArvMountPoint)
1260                 cr.CleanupDirs()
1261                 checkEmpty()
1262         }
1263
1264         for _, test := range []struct {
1265                 in  interface{}
1266                 out string
1267         }{
1268                 {in: "foo", out: `foo`},
1269                 {in: nil, out: "error"},
1270                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1271         } {
1272                 i = 0
1273                 cr.ArvMountPoint = ""
1274                 cr.Container.Mounts = map[string]arvados.Mount{
1275                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1276                 }
1277                 bindmounts, err := cr.SetupMounts()
1278                 if test.out == "error" {
1279                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1280                 } else {
1281                         c.Check(err, IsNil)
1282                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1283                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1284                         })
1285                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1286                         c.Check(err, IsNil)
1287                         c.Check(content, DeepEquals, []byte(test.out))
1288                 }
1289                 os.RemoveAll(cr.ArvMountPoint)
1290                 cr.CleanupDirs()
1291                 checkEmpty()
1292         }
1293
1294         // Read-only mount points are allowed underneath output_dir mount point
1295         {
1296                 i = 0
1297                 cr.ArvMountPoint = ""
1298                 cr.Container.Mounts = make(map[string]arvados.Mount)
1299                 cr.Container.Mounts = map[string]arvados.Mount{
1300                         "/tmp":     {Kind: "tmp"},
1301                         "/tmp/foo": {Kind: "collection"},
1302                 }
1303                 cr.Container.OutputPath = "/tmp"
1304
1305                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1306
1307                 bindmounts, err := cr.SetupMounts()
1308                 c.Check(err, IsNil)
1309                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1310                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1311                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1312                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1313                         "/tmp":     {realTemp + "/tmp2", false},
1314                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1315                 })
1316                 os.RemoveAll(cr.ArvMountPoint)
1317                 cr.CleanupDirs()
1318                 checkEmpty()
1319         }
1320
1321         // Writable mount points copied to output_dir mount point
1322         {
1323                 i = 0
1324                 cr.ArvMountPoint = ""
1325                 cr.Container.Mounts = make(map[string]arvados.Mount)
1326                 cr.Container.Mounts = map[string]arvados.Mount{
1327                         "/tmp": {Kind: "tmp"},
1328                         "/tmp/foo": {Kind: "collection",
1329                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1330                                 Writable:         true},
1331                         "/tmp/bar": {Kind: "collection",
1332                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1333                                 Path:             "baz",
1334                                 Writable:         true},
1335                 }
1336                 cr.Container.OutputPath = "/tmp"
1337
1338                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1339                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1340
1341                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1342                 rf.Write([]byte("bar"))
1343                 rf.Close()
1344
1345                 _, err := cr.SetupMounts()
1346                 c.Check(err, IsNil)
1347                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1348                 c.Check(err, IsNil)
1349                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1350                 c.Check(err, IsNil)
1351                 os.RemoveAll(cr.ArvMountPoint)
1352                 cr.CleanupDirs()
1353                 checkEmpty()
1354         }
1355
1356         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1357         {
1358                 i = 0
1359                 cr.ArvMountPoint = ""
1360                 cr.Container.Mounts = make(map[string]arvados.Mount)
1361                 cr.Container.Mounts = map[string]arvados.Mount{
1362                         "/tmp":     {Kind: "tmp"},
1363                         "/tmp/foo": {Kind: "tmp"},
1364                 }
1365                 cr.Container.OutputPath = "/tmp"
1366
1367                 _, err := cr.SetupMounts()
1368                 c.Check(err, NotNil)
1369                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1370                 os.RemoveAll(cr.ArvMountPoint)
1371                 cr.CleanupDirs()
1372                 checkEmpty()
1373         }
1374
1375         // Only mount point of kind 'collection' is allowed for stdin
1376         {
1377                 i = 0
1378                 cr.ArvMountPoint = ""
1379                 cr.Container.Mounts = make(map[string]arvados.Mount)
1380                 cr.Container.Mounts = map[string]arvados.Mount{
1381                         "stdin": {Kind: "tmp"},
1382                 }
1383
1384                 _, err := cr.SetupMounts()
1385                 c.Check(err, NotNil)
1386                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1387                 os.RemoveAll(cr.ArvMountPoint)
1388                 cr.CleanupDirs()
1389                 checkEmpty()
1390         }
1391
1392         // git_tree mounts
1393         {
1394                 i = 0
1395                 cr.ArvMountPoint = ""
1396                 (*GitMountSuite)(nil).useTestGitServer(c)
1397                 cr.token = arvadostest.ActiveToken
1398                 cr.Container.Mounts = make(map[string]arvados.Mount)
1399                 cr.Container.Mounts = map[string]arvados.Mount{
1400                         "/tip": {
1401                                 Kind:   "git_tree",
1402                                 UUID:   arvadostest.Repository2UUID,
1403                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1404                                 Path:   "/",
1405                         },
1406                         "/non-tip": {
1407                                 Kind:   "git_tree",
1408                                 UUID:   arvadostest.Repository2UUID,
1409                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1410                                 Path:   "/",
1411                         },
1412                 }
1413                 cr.Container.OutputPath = "/tmp"
1414
1415                 bindmounts, err := cr.SetupMounts()
1416                 c.Check(err, IsNil)
1417
1418                 for path, mount := range bindmounts {
1419                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1420                 }
1421
1422                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1423                 c.Check(err, IsNil)
1424                 c.Check(string(data), Equals, "\000\001\002\003")
1425                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1426                 c.Check(err, FitsTypeOf, &os.PathError{})
1427                 c.Check(os.IsNotExist(err), Equals, true)
1428
1429                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1430                 c.Check(err, IsNil)
1431                 c.Check(string(data), Equals, "\000\001\002\003")
1432                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1433                 c.Check(err, IsNil)
1434                 c.Check(string(data), Equals, "testfile\n")
1435
1436                 cr.CleanupDirs()
1437                 checkEmpty()
1438         }
1439 }
1440
1441 func (s *TestSuite) TestStdout(c *C) {
1442         helperRecord := `{
1443                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1444                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1445                 "cwd": "/bin",
1446                 "environment": {"FROBIZ": "bilbo"},
1447                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1448                 "output_path": "/tmp",
1449                 "priority": 1,
1450                 "runtime_constraints": {},
1451                 "state": "Locked"
1452         }`
1453
1454         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1455                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1456         })
1457
1458         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1459         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1460         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1461 }
1462
1463 // Used by the TestStdoutWithWrongPath*()
1464 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1465         err := json.Unmarshal([]byte(record), &s.api.Container)
1466         c.Assert(err, IsNil)
1467         s.executor.runFunc = fn
1468         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1469         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1470                 return s.api, &KeepTestClient{}, nil, nil
1471         }
1472         return s.api, s.runner, s.runner.Run()
1473 }
1474
1475 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1476         _, _, err := s.stdoutErrorRunHelper(c, `{
1477     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1478     "output_path": "/tmp",
1479     "state": "Locked"
1480 }`, func() {})
1481         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1482 }
1483
1484 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1485         _, _, err := s.stdoutErrorRunHelper(c, `{
1486     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1487     "output_path": "/tmp",
1488     "state": "Locked"
1489 }`, func() {})
1490         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1491 }
1492
1493 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1494         _, _, err := s.stdoutErrorRunHelper(c, `{
1495     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1496     "output_path": "/tmp",
1497     "state": "Locked"
1498 }`, func() {})
1499         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1500 }
1501
1502 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1503         s.fullRunHelper(c, `{
1504     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1505     "container_image": "`+arvadostest.DockerImage112PDH+`",
1506     "cwd": "/bin",
1507     "environment": {},
1508     "mounts": {"/tmp": {"kind": "tmp"} },
1509     "output_path": "/tmp",
1510     "priority": 1,
1511     "runtime_constraints": {"API": true},
1512     "state": "Locked"
1513 }`, nil, 0, func() {
1514                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1515                 s.executor.exit <- 3
1516         })
1517         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1518         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1519 }
1520
1521 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1522         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1523         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1524         s.fullRunHelper(c, `{
1525     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1526     "container_image": "`+arvadostest.DockerImage112PDH+`",
1527     "cwd": "/bin",
1528     "environment": {},
1529     "mounts": {"/tmp": {"kind": "tmp"} },
1530     "output_path": "/tmp",
1531     "priority": 1,
1532     "runtime_constraints": {"API": true},
1533     "state": "Locked"
1534 }`, nil, 0, func() {
1535                 s.api.Container.Output = arvadostest.DockerImage112PDH
1536         })
1537
1538         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1539         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1540         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1541 }
1542
1543 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1544         helperRecord := `{
1545                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1546                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1547                 "cwd": "/bin",
1548                 "environment": {"FROBIZ": "bilbo"},
1549                 "mounts": {
1550         "/tmp": {"kind": "tmp"},
1551         "/tmp/foo": {"kind": "collection",
1552                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1553                      "exclude_from_output": true
1554         },
1555         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1556     },
1557                 "output_path": "/tmp",
1558                 "priority": 1,
1559                 "runtime_constraints": {},
1560                 "state": "Locked"
1561         }`
1562
1563         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1564
1565         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1566                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1567         })
1568
1569         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1570         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1571         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1572 }
1573
1574 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1575         helperRecord := `{
1576                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1577                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1578                 "cwd": "/bin",
1579                 "environment": {"FROBIZ": "bilbo"},
1580                 "mounts": {
1581         "/tmp": {"kind": "tmp"},
1582         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1583         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1584         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1585         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1586         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1587     },
1588                 "output_path": "/tmp",
1589                 "priority": 1,
1590                 "runtime_constraints": {},
1591                 "state": "Locked"
1592         }`
1593
1594         extraMounts := []string{
1595                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1596                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1597                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1598         }
1599
1600         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1601                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1602         })
1603
1604         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1605                 "/tmp":                   {realtemp + "/tmp1", false},
1606                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1607                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1608                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1609                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1610         })
1611
1612         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1613         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1614         for _, v := range api.Content {
1615                 if v["collection"] != nil {
1616                         c.Check(v["ensure_unique_name"], Equals, true)
1617                         collection := v["collection"].(arvadosclient.Dict)
1618                         if strings.Index(collection["name"].(string), "output") == 0 {
1619                                 manifest := collection["manifest_text"].(string)
1620
1621                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1622 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1623 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1624 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1625 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1626 `)
1627                         }
1628                 }
1629         }
1630 }
1631
1632 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1633         helperRecord := `{
1634                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1635                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1636                 "cwd": "/bin",
1637                 "environment": {"FROBIZ": "bilbo"},
1638                 "mounts": {
1639         "/tmp": {"kind": "tmp"},
1640         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1641         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1642     },
1643                 "output_path": "/tmp",
1644                 "priority": 1,
1645                 "runtime_constraints": {},
1646                 "state": "Locked"
1647         }`
1648
1649         extraMounts := []string{
1650                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1651         }
1652
1653         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1654                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1655         })
1656
1657         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1658         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1659         for _, v := range s.api.Content {
1660                 if v["collection"] != nil {
1661                         collection := v["collection"].(arvadosclient.Dict)
1662                         if strings.Index(collection["name"].(string), "output") == 0 {
1663                                 manifest := collection["manifest_text"].(string)
1664
1665                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1666 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1667 `)
1668                         }
1669                 }
1670         }
1671 }
1672
1673 func (s *TestSuite) TestOutputError(c *C) {
1674         helperRecord := `{
1675                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1676                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1677                 "cwd": "/bin",
1678                 "environment": {"FROBIZ": "bilbo"},
1679                 "mounts": {
1680                         "/tmp": {"kind": "tmp"}
1681                 },
1682                 "output_path": "/tmp",
1683                 "priority": 1,
1684                 "runtime_constraints": {},
1685                 "state": "Locked"
1686         }`
1687         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1688                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1689         })
1690
1691         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1692 }
1693
1694 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1695         helperRecord := `{
1696                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1697                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1698                 "cwd": "/bin",
1699                 "environment": {"FROBIZ": "bilbo"},
1700                 "mounts": {
1701         "/tmp": {"kind": "tmp"},
1702         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1703         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1704     },
1705                 "output_path": "/tmp",
1706                 "priority": 1,
1707                 "runtime_constraints": {},
1708                 "state": "Locked"
1709         }`
1710
1711         extraMounts := []string{
1712                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1713         }
1714
1715         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1716                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1717         })
1718
1719         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1720         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1721         for _, v := range api.Content {
1722                 if v["collection"] != nil {
1723                         collection := v["collection"].(arvadosclient.Dict)
1724                         if strings.Index(collection["name"].(string), "output") == 0 {
1725                                 manifest := collection["manifest_text"].(string)
1726                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1727 `)
1728                         }
1729                 }
1730         }
1731 }
1732
1733 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1734         helperRecord := `{
1735                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1736                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1737                 "cwd": "/bin",
1738                 "environment": {"FROBIZ": "bilbo"},
1739                 "mounts": {
1740         "/tmp": {"kind": "tmp"},
1741         "stdin": {"kind": "json", "content": "foo"},
1742         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1743     },
1744                 "output_path": "/tmp",
1745                 "priority": 1,
1746                 "runtime_constraints": {},
1747                 "state": "Locked"
1748         }`
1749
1750         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1751                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1752         })
1753
1754         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1755         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1756         for _, v := range api.Content {
1757                 if v["collection"] != nil {
1758                         collection := v["collection"].(arvadosclient.Dict)
1759                         if strings.Index(collection["name"].(string), "output") == 0 {
1760                                 manifest := collection["manifest_text"].(string)
1761                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1762 `)
1763                         }
1764                 }
1765         }
1766 }
1767
1768 func (s *TestSuite) TestStderrMount(c *C) {
1769         api, cr, _ := s.fullRunHelper(c, `{
1770     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1771     "container_image": "`+arvadostest.DockerImage112PDH+`",
1772     "cwd": ".",
1773     "environment": {},
1774     "mounts": {"/tmp": {"kind": "tmp"},
1775                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1776                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1777     "output_path": "/tmp",
1778     "priority": 1,
1779     "runtime_constraints": {},
1780     "state": "Locked"
1781 }`, nil, 1, func() {
1782                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1783                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1784         })
1785
1786         final := api.CalledWith("container.state", "Complete")
1787         c.Assert(final, NotNil)
1788         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1789         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1790
1791         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1792 }
1793
1794 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1795         s.api.callraw = true
1796         err := s.runner.fetchContainerRecord()
1797         c.Assert(err, IsNil)
1798         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1799         c.Logf("%#v", s.runner.Container)
1800         c.Check(err, IsNil)
1801         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1802 }
1803
1804 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1805         nextState := ""
1806         for _, setup := range []func(){
1807                 func() {
1808                         c.Log("// waitErr = ocl runtime error")
1809                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1810                         nextState = "Cancelled"
1811                 },
1812                 func() {
1813                         c.Log("// loadErr = cannot connect")
1814                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1815                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1816                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1817                         c.Assert(err, IsNil)
1818                         nextState = "Queued"
1819                 },
1820         } {
1821                 s.SetUpTest(c)
1822                 setup()
1823                 s.fullRunHelper(c, `{
1824     "command": ["echo", "hello world"],
1825     "container_image": "`+arvadostest.DockerImage112PDH+`",
1826     "cwd": ".",
1827     "environment": {},
1828     "mounts": {"/tmp": {"kind": "tmp"} },
1829     "output_path": "/tmp",
1830     "priority": 1,
1831     "runtime_constraints": {},
1832     "state": "Locked"
1833 }`, nil, 0, func() {})
1834                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1835                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1836                 if *brokenNodeHook != "" {
1837                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1838                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1839                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1840                 } else {
1841                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1842                 }
1843         }
1844 }
1845
1846 func (s *TestSuite) TestBadCommand(c *C) {
1847         for _, startError := range []string{
1848                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1849                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1850                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1851         } {
1852                 s.SetUpTest(c)
1853                 s.executor.startErr = errors.New(startError)
1854                 s.fullRunHelper(c, `{
1855     "command": ["echo", "hello world"],
1856     "container_image": "`+arvadostest.DockerImage112PDH+`",
1857     "cwd": ".",
1858     "environment": {},
1859     "mounts": {"/tmp": {"kind": "tmp"} },
1860     "output_path": "/tmp",
1861     "priority": 1,
1862     "runtime_constraints": {},
1863     "state": "Locked"
1864 }`, nil, 0, func() {})
1865                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1866                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1867         }
1868 }
1869
1870 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1871         helperRecord := `{
1872                 "command": ["true"],
1873                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1874                 "cwd": "/bin",
1875                 "mounts": {
1876                     "/tmp": {"kind": "tmp"},
1877                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1878                 },
1879                 "secret_mounts": {
1880                 },
1881                 "output_path": "/tmp",
1882                 "priority": 1,
1883                 "runtime_constraints": {},
1884                 "state": "Locked"
1885         }`
1886
1887         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1888                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1889                 c.Check(err, IsNil)
1890                 c.Check(string(content), Equals, "mypassword")
1891         })
1892
1893         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1894         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1895         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1896         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1897
1898         // under secret mounts, not captured in output
1899         helperRecord = `{
1900                 "command": ["true"],
1901                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1902                 "cwd": "/bin",
1903                 "mounts": {
1904                     "/tmp": {"kind": "tmp"}
1905                 },
1906                 "secret_mounts": {
1907                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1908                 },
1909                 "output_path": "/tmp",
1910                 "priority": 1,
1911                 "runtime_constraints": {},
1912                 "state": "Locked"
1913         }`
1914
1915         s.SetUpTest(c)
1916         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1917                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1918                 c.Check(err, IsNil)
1919                 c.Check(string(content), Equals, "mypassword")
1920         })
1921
1922         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1923         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1924         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1925         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1926 }
1927
1928 type FakeProcess struct {
1929         cmdLine []string
1930 }
1931
1932 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1933         return fp.cmdLine, nil
1934 }