lib/crunchrun/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "errors"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "regexp"
  18         "runtime/pprof"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/manifest"
  29         "golang.org/x/net/context"
  30
  31         . "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func TestCrunchExec(t *testing.T) {
  36         TestingT(t)
  37 }
  38
  39 var _ = Suite(&TestSuite{})
  40
  41 type TestSuite struct {
  42         client                   *arvados.Client
  43         api                      *ArvTestClient
  44         runner                   *ContainerRunner
  45         executor                 *stubExecutor
  46         keepmount                string
  47         testDispatcherKeepClient KeepTestClient
  48         testContainerKeepClient  KeepTestClient
  49 }
  50
  51 func (s *TestSuite) SetUpTest(c *C) {
  52         *brokenNodeHook = ""
  53         s.client = arvados.NewClientFromEnv()
  54         s.executor = &stubExecutor{}
  55         var err error
  56         s.api = &ArvTestClient{}
  57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
  58         c.Assert(err, IsNil)
  59         s.runner.executor = s.executor
  60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
  61                 return s.api, &s.testContainerKeepClient, s.client, nil
  62         }
  63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
  64                 s.runner.ArvMountPoint = s.keepmount
  65                 return nil, nil
  66         }
  67         s.keepmount = c.MkDir()
  68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
  69         c.Assert(err, IsNil)
  70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
  71         c.Assert(err, IsNil)
  72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
  73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
  74         c.Assert(err, IsNil)
  75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
  76         c.Assert(err, IsNil)
  77         s.runner.ArvMountPoint = s.keepmount
  78 }
  79
  80 type ArvTestClient struct {
  81         Total   int64
  82         Calls   int
  83         Content []arvadosclient.Dict
  84         arvados.Container
  85         secretMounts []byte
  86         Logs         map[string]*bytes.Buffer
  87         sync.Mutex
  88         WasSetRunning bool
  89         callraw       bool
  90 }
  91
  92 type KeepTestClient struct {
  93         Called         bool
  94         Content        []byte
  95         StorageClasses []string
  96 }
  97
  98 type stubExecutor struct {
  99         imageLoaded bool
 100         loaded      string
 101         loadErr     error
 102         exitCode    int
 103         createErr   error
 104         created     containerSpec
 105         startErr    error
 106         waitSleep   time.Duration
 107         waitErr     error
 108         stopErr     error
 109         stopped     bool
 110         closed      bool
 111         runFunc     func()
 112         exit        chan int
 113 }
 114
 115 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
 116         containerClient *arvados.Client) error {
 117         e.loaded = tarball
 118         return e.loadErr
 119 }
 120 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
 121 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
 122 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
 123 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
 124 func (e *stubExecutor) Close()                          { e.closed = true }
 125 func (e *stubExecutor) Wait(context.Context) (int, error) {
 126         return <-e.exit, e.waitErr
 127 }
 128
 129 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
 130
 131 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
 132 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
 133 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
 134
 135 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 136 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 137
 138 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
 139 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
 140 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
 141 `
 142
 143 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 144
 145 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
 146 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
 147
 148 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
 149 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
 150
 151 func (client *ArvTestClient) Create(resourceType string,
 152         parameters arvadosclient.Dict,
 153         output interface{}) error {
 154
 155         client.Mutex.Lock()
 156         defer client.Mutex.Unlock()
 157
 158         client.Calls++
 159         client.Content = append(client.Content, parameters)
 160
 161         if resourceType == "logs" {
 162                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 163                 if client.Logs == nil {
 164                         client.Logs = make(map[string]*bytes.Buffer)
 165                 }
 166                 if client.Logs[et] == nil {
 167                         client.Logs[et] = &bytes.Buffer{}
 168                 }
 169                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 170         }
 171
 172         if resourceType == "collections" && output != nil {
 173                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 174                 outmap := output.(*arvados.Collection)
 175                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 176                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
 177         }
 178
 179         return nil
 180 }
 181
 182 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 183         switch {
 184         case method == "GET" && resourceType == "containers" && action == "auth":
 185                 return json.Unmarshal([]byte(`{
 186                         "kind": "arvados#api_client_authorization",
 187                         "uuid": "`+fakeAuthUUID+`",
 188                         "api_token": "`+fakeAuthToken+`"
 189                         }`), output)
 190         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
 191                 if client.secretMounts != nil {
 192                         return json.Unmarshal(client.secretMounts, output)
 193                 }
 194                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
 195         default:
 196                 return fmt.Errorf("Not found")
 197         }
 198 }
 199
 200 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 201         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 202         var j []byte
 203         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
 204                 j = []byte(`{
 205                         "kind": "arvados#nodeList",
 206                         "items": [{
 207                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
 208                                 "hostname": "compute2",
 209                                 "properties": {"total_cpu_cores": 16}
 210                         }]}`)
 211         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 212                 if uuid == "" {
 213                         j, err = json.Marshal(map[string]interface{}{
 214                                 "items": []interface{}{client.Container},
 215                                 "kind":  "arvados#nodeList",
 216                         })
 217                 } else {
 218                         j, err = json.Marshal(client.Container)
 219                 }
 220         } else {
 221                 j = []byte(`{
 222                         "command": ["sleep", "1"],
 223                         "container_image": "` + arvadostest.DockerImage112PDH + `",
 224                         "cwd": ".",
 225                         "environment": {},
 226                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 227                         "output_path": "/tmp",
 228                         "priority": 1,
 229                         "runtime_constraints": {}
 230                 }`)
 231         }
 232         return ioutil.NopCloser(bytes.NewReader(j)), err
 233 }
 234
 235 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 236         if resourceType == "collections" {
 237                 if uuid == hwPDH {
 238                         output.(*arvados.Collection).ManifestText = hwManifest
 239                 } else if uuid == otherPDH {
 240                         output.(*arvados.Collection).ManifestText = otherManifest
 241                 } else if uuid == normalizedWithSubdirsPDH {
 242                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 243                 } else if uuid == denormalizedWithSubdirsPDH {
 244                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 245                 }
 246         }
 247         if resourceType == "containers" {
 248                 (*output.(*arvados.Container)) = client.Container
 249         }
 250         return nil
 251 }
 252
 253 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 254         client.Mutex.Lock()
 255         defer client.Mutex.Unlock()
 256         client.Calls++
 257         client.Content = append(client.Content, parameters)
 258         if resourceType == "containers" {
 259                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 260                         client.WasSetRunning = true
 261                 }
 262         } else if resourceType == "collections" {
 263                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 264                 output.(*arvados.Collection).UUID = uuid
 265                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
 266         }
 267         return nil
 268 }
 269
 270 var discoveryMap = map[string]interface{}{
 271         "defaultTrashLifetime":               float64(1209600),
 272         "crunchLimitLogBytesPerJob":          float64(67108864),
 273         "crunchLogThrottleBytes":             float64(65536),
 274         "crunchLogThrottlePeriod":            float64(60),
 275         "crunchLogThrottleLines":             float64(1024),
 276         "crunchLogPartialLineThrottlePeriod": float64(5),
 277         "crunchLogBytesPerEvent":             float64(4096),
 278         "crunchLogSecondsBetweenEvents":      float64(1),
 279 }
 280
 281 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 282         return discoveryMap[key], nil
 283 }
 284
 285 // CalledWith returns the parameters from the first API call whose
 286 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 287 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 288 // no call matches, it returns nil.
 289 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 290 call:
 291         for _, content := range client.Content {
 292                 var v interface{} = content
 293                 for _, k := range strings.Split(jpath, ".") {
 294                         if dict, ok := v.(arvadosclient.Dict); !ok {
 295                                 continue call
 296                         } else {
 297                                 v = dict[k]
 298                         }
 299                 }
 300                 if v == expect {
 301                         return content
 302                 }
 303         }
 304         return nil
 305 }
 306
 307 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 308         return locator, nil
 309 }
 310
 311 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 312         client.Content = opts.Data
 313         return arvados.BlockWriteResponse{
 314                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
 315         }, nil
 316 }
 317
 318 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
 319         return 0, errors.New("not implemented")
 320 }
 321
 322 func (client *KeepTestClient) ClearBlockCache() {
 323 }
 324
 325 func (client *KeepTestClient) Close() {
 326         client.Content = nil
 327 }
 328
 329 func (client *KeepTestClient) SetStorageClasses(sc []string) {
 330         client.StorageClasses = sc
 331 }
 332
 333 type FileWrapper struct {
 334         io.ReadCloser
 335         len int64
 336 }
 337
 338 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 339         return nil, errors.New("not implemented")
 340 }
 341
 342 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 343         return 0, errors.New("not implemented")
 344 }
 345
 346 func (fw FileWrapper) Size() int64 {
 347         return fw.len
 348 }
 349
 350 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 351         return nil, errors.New("not implemented")
 352 }
 353
 354 func (fw FileWrapper) Truncate(int64) error {
 355         return errors.New("not implemented")
 356 }
 357
 358 func (fw FileWrapper) Write([]byte) (int, error) {
 359         return 0, errors.New("not implemented")
 360 }
 361
 362 func (fw FileWrapper) Sync() error {
 363         return errors.New("not implemented")
 364 }
 365
 366 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 367         if filename == hwImageID+".tar" {
 368                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 369                 client.Called = true
 370                 return FileWrapper{rdr, 1321984}, nil
 371         } else if filename == "/file1_in_main.txt" {
 372                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 373                 client.Called = true
 374                 return FileWrapper{rdr, 3}, nil
 375         }
 376         return nil, nil
 377 }
 378
 379 func (s *TestSuite) TestLoadImage(c *C) {
 380         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 381         s.runner.Container.Mounts = map[string]arvados.Mount{
 382                 "/out": {Kind: "tmp", Writable: true},
 383         }
 384         s.runner.Container.OutputPath = "/out"
 385
 386         _, err := s.runner.SetupMounts()
 387         c.Assert(err, IsNil)
 388
 389         imageID, err := s.runner.LoadImage()
 390         c.Check(err, IsNil)
 391         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 392         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
 393
 394         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 395         s.executor.imageLoaded = false
 396         s.executor.loaded = ""
 397         s.executor.loadErr = errors.New("bork")
 398         imageID, err = s.runner.LoadImage()
 399         c.Check(err, ErrorMatches, ".*bork")
 400         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
 401
 402         s.runner.Container.ContainerImage = fakeInputCollectionPDH
 403         s.executor.imageLoaded = false
 404         s.executor.loaded = ""
 405         s.executor.loadErr = nil
 406         imageID, err = s.runner.LoadImage()
 407         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
 408         c.Check(s.executor.loaded, Equals, "")
 409 }
 410
 411 type ArvErrorTestClient struct{}
 412
 413 func (ArvErrorTestClient) Create(resourceType string,
 414         parameters arvadosclient.Dict,
 415         output interface{}) error {
 416         return nil
 417 }
 418
 419 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 420         if method == "GET" && resourceType == "containers" && action == "auth" {
 421                 return nil
 422         }
 423         return errors.New("ArvError")
 424 }
 425
 426 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 427         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 428         return nil, errors.New("ArvError")
 429 }
 430
 431 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 432         return errors.New("ArvError")
 433 }
 434
 435 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 436         return nil
 437 }
 438
 439 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 440         return discoveryMap[key], nil
 441 }
 442
 443 type KeepErrorTestClient struct {
 444         KeepTestClient
 445 }
 446
 447 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
 448         return nil, errors.New("KeepError")
 449 }
 450
 451 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
 452         return arvados.BlockWriteResponse{}, errors.New("KeepError")
 453 }
 454
 455 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 456         return "", errors.New("KeepError")
 457 }
 458
 459 type KeepReadErrorTestClient struct {
 460         KeepTestClient
 461 }
 462
 463 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
 464         return 0, errors.New("KeepError")
 465 }
 466
 467 type ErrorReader struct {
 468         FileWrapper
 469 }
 470
 471 func (ErrorReader) Read(p []byte) (n int, err error) {
 472         return 0, errors.New("ErrorReader")
 473 }
 474
 475 func (ErrorReader) Seek(int64, int) (int64, error) {
 476         return 0, errors.New("ErrorReader")
 477 }
 478
 479 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 480         return ErrorReader{}, nil
 481 }
 482
 483 type ClosableBuffer struct {
 484         bytes.Buffer
 485 }
 486
 487 func (*ClosableBuffer) Close() error {
 488         return nil
 489 }
 490
 491 type TestLogs struct {
 492         Stdout ClosableBuffer
 493         Stderr ClosableBuffer
 494 }
 495
 496 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
 497         if logstr == "stdout" {
 498                 return &tl.Stdout, nil
 499         }
 500         if logstr == "stderr" {
 501                 return &tl.Stderr, nil
 502         }
 503         return nil, errors.New("???")
 504 }
 505
 506 func dockerLog(fd byte, msg string) []byte {
 507         by := []byte(msg)
 508         header := make([]byte, 8+len(by))
 509         header[0] = fd
 510         header[7] = byte(len(by))
 511         copy(header[8:], by)
 512         return header
 513 }
 514
 515 func (s *TestSuite) TestRunContainer(c *C) {
 516         s.executor.runFunc = func() {
 517                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
 518                 s.executor.exit <- 0
 519         }
 520
 521         var logs TestLogs
 522         s.runner.NewLogWriter = logs.NewTestLoggingWriter
 523         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
 524         s.runner.Container.Command = []string{"./hw"}
 525         s.runner.Container.OutputStorageClasses = []string{"default"}
 526
 527         imageID, err := s.runner.LoadImage()
 528         c.Assert(err, IsNil)
 529
 530         err = s.runner.CreateContainer(imageID, nil)
 531         c.Assert(err, IsNil)
 532
 533         err = s.runner.StartContainer()
 534         c.Assert(err, IsNil)
 535
 536         err = s.runner.WaitFinish()
 537         c.Assert(err, IsNil)
 538
 539         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
 540         c.Check(logs.Stderr.String(), Equals, "")
 541 }
 542
 543 func (s *TestSuite) TestCommitLogs(c *C) {
 544         api := &ArvTestClient{}
 545         kc := &KeepTestClient{}
 546         defer kc.Close()
 547         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 548         c.Assert(err, IsNil)
 549         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 550
 551         cr.CrunchLog.Print("Hello world!")
 552         cr.CrunchLog.Print("Goodbye")
 553         cr.finalState = "Complete"
 554
 555         err = cr.CommitLogs()
 556         c.Check(err, IsNil)
 557
 558         c.Check(api.Calls, Equals, 2)
 559         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 560         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 561         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 562         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 563 }
 564
 565 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 566         api := &ArvTestClient{}
 567         kc := &KeepTestClient{}
 568         defer kc.Close()
 569         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 570         c.Assert(err, IsNil)
 571
 572         err = cr.UpdateContainerRunning()
 573         c.Check(err, IsNil)
 574
 575         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 576 }
 577
 578 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 579         api := &ArvTestClient{}
 580         kc := &KeepTestClient{}
 581         defer kc.Close()
 582         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 583         c.Assert(err, IsNil)
 584
 585         cr.LogsPDH = new(string)
 586         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 587
 588         cr.ExitCode = new(int)
 589         *cr.ExitCode = 42
 590         cr.finalState = "Complete"
 591
 592         err = cr.UpdateContainerFinal()
 593         c.Check(err, IsNil)
 594
 595         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 596         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 597         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 598 }
 599
 600 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 601         api := &ArvTestClient{}
 602         kc := &KeepTestClient{}
 603         defer kc.Close()
 604         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 605         c.Assert(err, IsNil)
 606         cr.cCancelled = true
 607         cr.finalState = "Cancelled"
 608
 609         err = cr.UpdateContainerFinal()
 610         c.Check(err, IsNil)
 611
 612         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 613         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 615 }
 616
 617 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 618 // dress rehearsal of the Run() function, starting from a JSON container record.
 619 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
 620         err := json.Unmarshal([]byte(record), &s.api.Container)
 621         c.Assert(err, IsNil)
 622         initialState := s.api.Container.State
 623
 624         var sm struct {
 625                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
 626         }
 627         err = json.Unmarshal([]byte(record), &sm)
 628         c.Check(err, IsNil)
 629         secretMounts, err := json.Marshal(sm)
 630         c.Assert(err, IsNil)
 631         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
 632
 633         s.executor.runFunc = func() {
 634                 fn()
 635                 s.executor.exit <- exitCode
 636         }
 637
 638         s.runner.statInterval = 100 * time.Millisecond
 639         s.runner.containerWatchdogInterval = time.Second
 640         am := &ArvMountCmdLine{}
 641         s.runner.RunArvMount = am.ArvMountTest
 642
 643         realTemp := c.MkDir()
 644         tempcount := 0
 645         s.runner.MkTempDir = func(_, prefix string) (string, error) {
 646                 tempcount++
 647                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 648                 err := os.Mkdir(d, os.ModePerm)
 649                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 650                         // Test case must have pre-populated the tempdir
 651                         err = nil
 652                 }
 653                 return d, err
 654         }
 655         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
 656                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
 657         }
 658
 659         if extraMounts != nil && len(extraMounts) > 0 {
 660                 err := s.runner.SetupArvMountPoint("keep")
 661                 c.Check(err, IsNil)
 662
 663                 for _, m := range extraMounts {
 664                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 665                 }
 666         }
 667
 668         err = s.runner.Run()
 669         if s.api.CalledWith("container.state", "Complete") != nil {
 670                 c.Check(err, IsNil)
 671         }
 672         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
 673                 c.Check(s.api.WasSetRunning, Equals, true)
 674                 var lastupdate arvadosclient.Dict
 675                 for _, content := range s.api.Content {
 676                         if content["container"] != nil {
 677                                 lastupdate = content["container"].(arvadosclient.Dict)
 678                         }
 679                 }
 680                 if lastupdate["log"] == nil {
 681                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
 682                 }
 683         }
 684
 685         if err != nil {
 686                 for k, v := range s.api.Logs {
 687                         c.Log(k)
 688                         c.Log(v.String())
 689                 }
 690         }
 691
 692         return s.api, s.runner, realTemp
 693 }
 694
 695 func (s *TestSuite) TestFullRunHello(c *C) {
 696         s.runner.enableMemoryLimit = true
 697         s.runner.networkMode = "default"
 698         s.fullRunHelper(c, `{
 699     "command": ["echo", "hello world"],
 700     "container_image": "`+arvadostest.DockerImage112PDH+`",
 701     "cwd": ".",
 702     "environment": {"foo":"bar","baz":"waz"},
 703     "mounts": {"/tmp": {"kind": "tmp"} },
 704     "output_path": "/tmp",
 705     "priority": 1,
 706     "runtime_constraints": {"vcpus":1,"ram":1000000},
 707     "state": "Locked",
 708     "output_storage_classes": ["default"]
 709 }`, nil, 0, func() {
 710                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
 711                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
 712                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
 713                 c.Check(s.executor.created.VCPUs, Equals, 1)
 714                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
 715                 c.Check(s.executor.created.NetworkMode, Equals, "default")
 716                 c.Check(s.executor.created.EnableNetwork, Equals, false)
 717                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
 718         })
 719
 720         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 721         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 722         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
 723         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
 724         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
 725 }
 726
 727 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
 728         var ran bool
 729         s.fullRunHelper(c, `{
 730     "command": ["sleep", "3"],
 731     "container_image": "`+arvadostest.DockerImage112PDH+`",
 732     "cwd": ".",
 733     "environment": {},
 734     "mounts": {"/tmp": {"kind": "tmp"} },
 735     "output_path": "/tmp",
 736     "priority": 1,
 737     "runtime_constraints": {},
 738     "scheduling_parameters":{"max_run_time": 1},
 739     "state": "Running"
 740 }`, nil, 2, func() {
 741                 ran = true
 742         })
 743         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
 744         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
 745         c.Check(ran, Equals, false)
 746 }
 747
 748 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 749         s.fullRunHelper(c, `{
 750     "command": ["sleep", "3"],
 751     "container_image": "`+arvadostest.DockerImage112PDH+`",
 752     "cwd": ".",
 753     "environment": {},
 754     "mounts": {"/tmp": {"kind": "tmp"} },
 755     "output_path": "/tmp",
 756     "priority": 1,
 757     "runtime_constraints": {},
 758     "scheduling_parameters":{"max_run_time": 1},
 759     "state": "Locked"
 760 }`, nil, 0, func() {
 761                 time.Sleep(3 * time.Second)
 762         })
 763
 764         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 765         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
 766 }
 767
 768 func (s *TestSuite) TestContainerWaitFails(c *C) {
 769         s.fullRunHelper(c, `{
 770     "command": ["sleep", "3"],
 771     "container_image": "`+arvadostest.DockerImage112PDH+`",
 772     "cwd": ".",
 773     "mounts": {"/tmp": {"kind": "tmp"} },
 774     "output_path": "/tmp",
 775     "priority": 1,
 776     "state": "Locked"
 777 }`, nil, 0, func() {
 778                 s.executor.waitErr = errors.New("Container is not running")
 779         })
 780
 781         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
 782         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 783 }
 784
 785 func (s *TestSuite) TestCrunchstat(c *C) {
 786         s.fullRunHelper(c, `{
 787                 "command": ["sleep", "1"],
 788                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 789                 "cwd": ".",
 790                 "environment": {},
 791                 "mounts": {"/tmp": {"kind": "tmp"} },
 792                 "output_path": "/tmp",
 793                 "priority": 1,
 794                 "runtime_constraints": {},
 795                 "state": "Locked"
 796         }`, nil, 0, func() {
 797                 time.Sleep(time.Second)
 798         })
 799
 800         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 801         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 802
 803         // We didn't actually start a container, so crunchstat didn't
 804         // find accounting files and therefore didn't log any stats.
 805         // It should have logged a "can't find accounting files"
 806         // message after one poll interval, though, so we can confirm
 807         // it's alive:
 808         c.Assert(s.api.Logs["crunchstat"], NotNil)
 809         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 810
 811         // The "files never appeared" log assures us that we called
 812         // (*crunchstat.Reporter)Stop(), and that we set it up with
 813         // the correct container ID "abcde":
 814         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
 815 }
 816
 817 func (s *TestSuite) TestNodeInfoLog(c *C) {
 818         os.Setenv("SLURMD_NODENAME", "compute2")
 819         s.fullRunHelper(c, `{
 820                 "command": ["sleep", "1"],
 821                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 822                 "cwd": ".",
 823                 "environment": {},
 824                 "mounts": {"/tmp": {"kind": "tmp"} },
 825                 "output_path": "/tmp",
 826                 "priority": 1,
 827                 "runtime_constraints": {},
 828                 "state": "Locked"
 829         }`, nil, 0,
 830                 func() {
 831                         time.Sleep(time.Second)
 832                 })
 833
 834         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 835         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 836
 837         c.Assert(s.api.Logs["node"], NotNil)
 838         json := s.api.Logs["node"].String()
 839         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
 840         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
 841         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
 842
 843         c.Assert(s.api.Logs["node-info"], NotNil)
 844         json = s.api.Logs["node-info"].String()
 845         c.Check(json, Matches, `(?ms).*Host Information.*`)
 846         c.Check(json, Matches, `(?ms).*CPU Information.*`)
 847         c.Check(json, Matches, `(?ms).*Memory Information.*`)
 848         c.Check(json, Matches, `(?ms).*Disk Space.*`)
 849         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 850 }
 851
 852 func (s *TestSuite) TestContainerRecordLog(c *C) {
 853         s.fullRunHelper(c, `{
 854                 "command": ["sleep", "1"],
 855                 "container_image": "`+arvadostest.DockerImage112PDH+`",
 856                 "cwd": ".",
 857                 "environment": {},
 858                 "mounts": {"/tmp": {"kind": "tmp"} },
 859                 "output_path": "/tmp",
 860                 "priority": 1,
 861                 "runtime_constraints": {},
 862                 "state": "Locked"
 863         }`, nil, 0,
 864                 func() {
 865                         time.Sleep(time.Second)
 866                 })
 867
 868         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 869         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 870
 871         c.Assert(s.api.Logs["container"], NotNil)
 872         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 873 }
 874
 875 func (s *TestSuite) TestFullRunStderr(c *C) {
 876         s.fullRunHelper(c, `{
 877     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 878     "container_image": "`+arvadostest.DockerImage112PDH+`",
 879     "cwd": ".",
 880     "environment": {},
 881     "mounts": {"/tmp": {"kind": "tmp"} },
 882     "output_path": "/tmp",
 883     "priority": 1,
 884     "runtime_constraints": {},
 885     "state": "Locked"
 886 }`, nil, 1, func() {
 887                 fmt.Fprintln(s.executor.created.Stdout, "hello")
 888                 fmt.Fprintln(s.executor.created.Stderr, "world")
 889         })
 890
 891         final := s.api.CalledWith("container.state", "Complete")
 892         c.Assert(final, NotNil)
 893         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 894         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 895
 896         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
 897         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
 898 }
 899
 900 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 901         s.fullRunHelper(c, `{
 902     "command": ["pwd"],
 903     "container_image": "`+arvadostest.DockerImage112PDH+`",
 904     "cwd": ".",
 905     "environment": {},
 906     "mounts": {"/tmp": {"kind": "tmp"} },
 907     "output_path": "/tmp",
 908     "priority": 1,
 909     "runtime_constraints": {},
 910     "state": "Locked"
 911 }`, nil, 0, func() {
 912                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
 913         })
 914
 915         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 916         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 917         c.Log(s.api.Logs["stdout"])
 918         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
 919 }
 920
 921 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 922         s.fullRunHelper(c, `{
 923     "command": ["pwd"],
 924     "container_image": "`+arvadostest.DockerImage112PDH+`",
 925     "cwd": "/bin",
 926     "environment": {},
 927     "mounts": {"/tmp": {"kind": "tmp"} },
 928     "output_path": "/tmp",
 929     "priority": 1,
 930     "runtime_constraints": {},
 931     "state": "Locked"
 932 }`, nil, 0, func() {
 933                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 934         })
 935
 936         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 937         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 938         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 939 }
 940
 941 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
 942         s.fullRunHelper(c, `{
 943     "command": ["pwd"],
 944     "container_image": "`+arvadostest.DockerImage112PDH+`",
 945     "cwd": "/bin",
 946     "environment": {},
 947     "mounts": {"/tmp": {"kind": "tmp"} },
 948     "output_path": "/tmp",
 949     "priority": 1,
 950     "runtime_constraints": {},
 951     "state": "Locked",
 952     "output_storage_classes": ["foo", "bar"]
 953 }`, nil, 0, func() {
 954                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
 955         })
 956
 957         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
 958         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
 959         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
 960         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 961         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
 962 }
 963
 964 func (s *TestSuite) TestStopOnSignal(c *C) {
 965         s.executor.runFunc = func() {
 966                 s.executor.created.Stdout.Write([]byte("foo\n"))
 967                 s.runner.SigChan <- syscall.SIGINT
 968         }
 969         s.testStopContainer(c)
 970 }
 971
 972 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 973         s.executor.runFunc = func() {
 974                 s.executor.created.Stdout.Write([]byte("foo\n"))
 975                 s.runner.ArvMountExit <- nil
 976                 close(s.runner.ArvMountExit)
 977         }
 978         s.runner.ArvMountExit = make(chan error)
 979         s.testStopContainer(c)
 980 }
 981
 982 func (s *TestSuite) testStopContainer(c *C) {
 983         record := `{
 984     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 985     "container_image": "` + arvadostest.DockerImage112PDH + `",
 986     "cwd": ".",
 987     "environment": {},
 988     "mounts": {"/tmp": {"kind": "tmp"} },
 989     "output_path": "/tmp",
 990     "priority": 1,
 991     "runtime_constraints": {},
 992     "state": "Locked"
 993 }`
 994
 995         err := json.Unmarshal([]byte(record), &s.api.Container)
 996         c.Assert(err, IsNil)
 997
 998         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 999         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1000                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1001         }
1002
1003         done := make(chan error)
1004         go func() {
1005                 done <- s.runner.Run()
1006         }()
1007         select {
1008         case <-time.After(20 * time.Second):
1009                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1010                 c.Fatal("timed out")
1011         case err = <-done:
1012                 c.Check(err, IsNil)
1013         }
1014         for k, v := range s.api.Logs {
1015                 c.Log(k)
1016                 c.Log(v.String(), "\n")
1017         }
1018
1019         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1020         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1021         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1022 }
1023
1024 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1025         s.fullRunHelper(c, `{
1026     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1027     "container_image": "`+arvadostest.DockerImage112PDH+`",
1028     "cwd": "/bin",
1029     "environment": {"FROBIZ": "bilbo"},
1030     "mounts": {"/tmp": {"kind": "tmp"} },
1031     "output_path": "/tmp",
1032     "priority": 1,
1033     "runtime_constraints": {},
1034     "state": "Locked"
1035 }`, nil, 0, func() {
1036                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1037         })
1038
1039         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1040         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1041         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1042 }
1043
1044 type ArvMountCmdLine struct {
1045         Cmd   []string
1046         token string
1047 }
1048
1049 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1050         am.Cmd = c
1051         am.token = token
1052         return nil, nil
1053 }
1054
1055 func stubCert(temp string) string {
1056         path := temp + "/ca-certificates.crt"
1057         crt, _ := os.Create(path)
1058         crt.Close()
1059         arvadosclient.CertFiles = []string{path}
1060         return path
1061 }
1062
1063 func (s *TestSuite) TestSetupMounts(c *C) {
1064         cr := s.runner
1065         am := &ArvMountCmdLine{}
1066         cr.RunArvMount = am.ArvMountTest
1067         cr.ContainerArvClient = &ArvTestClient{}
1068         cr.ContainerKeepClient = &KeepTestClient{}
1069         cr.Container.OutputStorageClasses = []string{"default"}
1070
1071         realTemp := c.MkDir()
1072         certTemp := c.MkDir()
1073         stubCertPath := stubCert(certTemp)
1074         cr.parentTemp = realTemp
1075
1076         i := 0
1077         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1078                 i++
1079                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1080                 err := os.Mkdir(d, os.ModePerm)
1081                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1082                         // Test case must have pre-populated the tempdir
1083                         err = nil
1084                 }
1085                 return d, err
1086         }
1087
1088         checkEmpty := func() {
1089                 // Should be deleted.
1090                 _, err := os.Stat(realTemp)
1091                 c.Assert(os.IsNotExist(err), Equals, true)
1092
1093                 // Now recreate it for the next test.
1094                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1095         }
1096
1097         {
1098                 i = 0
1099                 cr.ArvMountPoint = ""
1100                 cr.Container.Mounts = make(map[string]arvados.Mount)
1101                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1102                 cr.Container.OutputPath = "/tmp"
1103                 cr.statInterval = 5 * time.Second
1104                 bindmounts, err := cr.SetupMounts()
1105                 c.Check(err, IsNil)
1106                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground", "--allow-other",
1107                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1108                         "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1109                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1110                 os.RemoveAll(cr.ArvMountPoint)
1111                 cr.CleanupDirs()
1112                 checkEmpty()
1113         }
1114
1115         {
1116                 i = 0
1117                 cr.ArvMountPoint = ""
1118                 cr.Container.Mounts = make(map[string]arvados.Mount)
1119                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1120                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1121                 cr.Container.OutputPath = "/out"
1122                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1123
1124                 bindmounts, err := cr.SetupMounts()
1125                 c.Check(err, IsNil)
1126                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground", "--allow-other",
1127                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1128                         "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1129                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1130                 os.RemoveAll(cr.ArvMountPoint)
1131                 cr.CleanupDirs()
1132                 checkEmpty()
1133         }
1134
1135         {
1136                 i = 0
1137                 cr.ArvMountPoint = ""
1138                 cr.Container.Mounts = make(map[string]arvados.Mount)
1139                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1140                 cr.Container.OutputPath = "/tmp"
1141                 cr.Container.RuntimeConstraints.API = true
1142                 cr.Container.OutputStorageClasses = []string{"default"}
1143
1144                 bindmounts, err := cr.SetupMounts()
1145                 c.Check(err, IsNil)
1146                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground", "--allow-other",
1147                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1148                         "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1149                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1150                 os.RemoveAll(cr.ArvMountPoint)
1151                 cr.CleanupDirs()
1152                 checkEmpty()
1153
1154                 cr.Container.RuntimeConstraints.API = false
1155         }
1156
1157         {
1158                 i = 0
1159                 cr.ArvMountPoint = ""
1160                 cr.Container.Mounts = map[string]arvados.Mount{
1161                         "/keeptmp": {Kind: "collection", Writable: true},
1162                 }
1163                 cr.Container.OutputPath = "/keeptmp"
1164
1165                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1166
1167                 bindmounts, err := cr.SetupMounts()
1168                 c.Check(err, IsNil)
1169                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground", "--allow-other",
1170                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1171                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1172                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1173                 os.RemoveAll(cr.ArvMountPoint)
1174                 cr.CleanupDirs()
1175                 checkEmpty()
1176         }
1177
1178         {
1179                 i = 0
1180                 cr.ArvMountPoint = ""
1181                 cr.Container.Mounts = map[string]arvados.Mount{
1182                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1183                         "/keepout": {Kind: "collection", Writable: true},
1184                 }
1185                 cr.Container.OutputPath = "/keepout"
1186
1187                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1188                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1189
1190                 bindmounts, err := cr.SetupMounts()
1191                 c.Check(err, IsNil)
1192                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground", "--allow-other",
1193                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1194                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1195                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1196                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1197                         "/keepout": {realTemp + "/keep1/tmp0", false},
1198                 })
1199                 os.RemoveAll(cr.ArvMountPoint)
1200                 cr.CleanupDirs()
1201                 checkEmpty()
1202         }
1203
1204         {
1205                 i = 0
1206                 cr.ArvMountPoint = ""
1207                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1208                 cr.Container.Mounts = map[string]arvados.Mount{
1209                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1210                         "/keepout": {Kind: "collection", Writable: true},
1211                 }
1212                 cr.Container.OutputPath = "/keepout"
1213
1214                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1215                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1216
1217                 bindmounts, err := cr.SetupMounts()
1218                 c.Check(err, IsNil)
1219                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground", "--allow-other",
1220                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1221                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1222                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1223                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1224                         "/keepout": {realTemp + "/keep1/tmp0", false},
1225                 })
1226                 os.RemoveAll(cr.ArvMountPoint)
1227                 cr.CleanupDirs()
1228                 checkEmpty()
1229         }
1230
1231         for _, test := range []struct {
1232                 in  interface{}
1233                 out string
1234         }{
1235                 {in: "foo", out: `"foo"`},
1236                 {in: nil, out: `null`},
1237                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1238         } {
1239                 i = 0
1240                 cr.ArvMountPoint = ""
1241                 cr.Container.Mounts = map[string]arvados.Mount{
1242                         "/mnt/test.json": {Kind: "json", Content: test.in},
1243                 }
1244                 bindmounts, err := cr.SetupMounts()
1245                 c.Check(err, IsNil)
1246                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1247                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1248                 })
1249                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1250                 c.Check(err, IsNil)
1251                 c.Check(content, DeepEquals, []byte(test.out))
1252                 os.RemoveAll(cr.ArvMountPoint)
1253                 cr.CleanupDirs()
1254                 checkEmpty()
1255         }
1256
1257         for _, test := range []struct {
1258                 in  interface{}
1259                 out string
1260         }{
1261                 {in: "foo", out: `foo`},
1262                 {in: nil, out: "error"},
1263                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1264         } {
1265                 i = 0
1266                 cr.ArvMountPoint = ""
1267                 cr.Container.Mounts = map[string]arvados.Mount{
1268                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1269                 }
1270                 bindmounts, err := cr.SetupMounts()
1271                 if test.out == "error" {
1272                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1273                 } else {
1274                         c.Check(err, IsNil)
1275                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1276                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1277                         })
1278                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1279                         c.Check(err, IsNil)
1280                         c.Check(content, DeepEquals, []byte(test.out))
1281                 }
1282                 os.RemoveAll(cr.ArvMountPoint)
1283                 cr.CleanupDirs()
1284                 checkEmpty()
1285         }
1286
1287         // Read-only mount points are allowed underneath output_dir mount point
1288         {
1289                 i = 0
1290                 cr.ArvMountPoint = ""
1291                 cr.Container.Mounts = make(map[string]arvados.Mount)
1292                 cr.Container.Mounts = map[string]arvados.Mount{
1293                         "/tmp":     {Kind: "tmp"},
1294                         "/tmp/foo": {Kind: "collection"},
1295                 }
1296                 cr.Container.OutputPath = "/tmp"
1297
1298                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1299
1300                 bindmounts, err := cr.SetupMounts()
1301                 c.Check(err, IsNil)
1302                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground", "--allow-other",
1303                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1304                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1305                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1306                         "/tmp":     {realTemp + "/tmp2", false},
1307                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1308                 })
1309                 os.RemoveAll(cr.ArvMountPoint)
1310                 cr.CleanupDirs()
1311                 checkEmpty()
1312         }
1313
1314         // Writable mount points copied to output_dir mount point
1315         {
1316                 i = 0
1317                 cr.ArvMountPoint = ""
1318                 cr.Container.Mounts = make(map[string]arvados.Mount)
1319                 cr.Container.Mounts = map[string]arvados.Mount{
1320                         "/tmp": {Kind: "tmp"},
1321                         "/tmp/foo": {Kind: "collection",
1322                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1323                                 Writable:         true},
1324                         "/tmp/bar": {Kind: "collection",
1325                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1326                                 Path:             "baz",
1327                                 Writable:         true},
1328                 }
1329                 cr.Container.OutputPath = "/tmp"
1330
1331                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1332                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1333
1334                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1335                 rf.Write([]byte("bar"))
1336                 rf.Close()
1337
1338                 _, err := cr.SetupMounts()
1339                 c.Check(err, IsNil)
1340                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1341                 c.Check(err, IsNil)
1342                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1343                 c.Check(err, IsNil)
1344                 os.RemoveAll(cr.ArvMountPoint)
1345                 cr.CleanupDirs()
1346                 checkEmpty()
1347         }
1348
1349         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1350         {
1351                 i = 0
1352                 cr.ArvMountPoint = ""
1353                 cr.Container.Mounts = make(map[string]arvados.Mount)
1354                 cr.Container.Mounts = map[string]arvados.Mount{
1355                         "/tmp":     {Kind: "tmp"},
1356                         "/tmp/foo": {Kind: "tmp"},
1357                 }
1358                 cr.Container.OutputPath = "/tmp"
1359
1360                 _, err := cr.SetupMounts()
1361                 c.Check(err, NotNil)
1362                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1363                 os.RemoveAll(cr.ArvMountPoint)
1364                 cr.CleanupDirs()
1365                 checkEmpty()
1366         }
1367
1368         // Only mount point of kind 'collection' is allowed for stdin
1369         {
1370                 i = 0
1371                 cr.ArvMountPoint = ""
1372                 cr.Container.Mounts = make(map[string]arvados.Mount)
1373                 cr.Container.Mounts = map[string]arvados.Mount{
1374                         "stdin": {Kind: "tmp"},
1375                 }
1376
1377                 _, err := cr.SetupMounts()
1378                 c.Check(err, NotNil)
1379                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1380                 os.RemoveAll(cr.ArvMountPoint)
1381                 cr.CleanupDirs()
1382                 checkEmpty()
1383         }
1384
1385         // git_tree mounts
1386         {
1387                 i = 0
1388                 cr.ArvMountPoint = ""
1389                 (*GitMountSuite)(nil).useTestGitServer(c)
1390                 cr.token = arvadostest.ActiveToken
1391                 cr.Container.Mounts = make(map[string]arvados.Mount)
1392                 cr.Container.Mounts = map[string]arvados.Mount{
1393                         "/tip": {
1394                                 Kind:   "git_tree",
1395                                 UUID:   arvadostest.Repository2UUID,
1396                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1397                                 Path:   "/",
1398                         },
1399                         "/non-tip": {
1400                                 Kind:   "git_tree",
1401                                 UUID:   arvadostest.Repository2UUID,
1402                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1403                                 Path:   "/",
1404                         },
1405                 }
1406                 cr.Container.OutputPath = "/tmp"
1407
1408                 bindmounts, err := cr.SetupMounts()
1409                 c.Check(err, IsNil)
1410
1411                 for path, mount := range bindmounts {
1412                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1413                 }
1414
1415                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1416                 c.Check(err, IsNil)
1417                 c.Check(string(data), Equals, "\000\001\002\003")
1418                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1419                 c.Check(err, FitsTypeOf, &os.PathError{})
1420                 c.Check(os.IsNotExist(err), Equals, true)
1421
1422                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1423                 c.Check(err, IsNil)
1424                 c.Check(string(data), Equals, "\000\001\002\003")
1425                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1426                 c.Check(err, IsNil)
1427                 c.Check(string(data), Equals, "testfile\n")
1428
1429                 cr.CleanupDirs()
1430                 checkEmpty()
1431         }
1432 }
1433
1434 func (s *TestSuite) TestStdout(c *C) {
1435         helperRecord := `{
1436                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1437                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1438                 "cwd": "/bin",
1439                 "environment": {"FROBIZ": "bilbo"},
1440                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1441                 "output_path": "/tmp",
1442                 "priority": 1,
1443                 "runtime_constraints": {},
1444                 "state": "Locked"
1445         }`
1446
1447         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1448                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1449         })
1450
1451         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1452         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1453         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1454 }
1455
1456 // Used by the TestStdoutWithWrongPath*()
1457 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1458         err := json.Unmarshal([]byte(record), &s.api.Container)
1459         c.Assert(err, IsNil)
1460         s.executor.runFunc = fn
1461         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1462         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1463                 return s.api, &KeepTestClient{}, nil, nil
1464         }
1465         return s.api, s.runner, s.runner.Run()
1466 }
1467
1468 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1469         _, _, err := s.stdoutErrorRunHelper(c, `{
1470     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1471     "output_path": "/tmp",
1472     "state": "Locked"
1473 }`, func() {})
1474         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1475 }
1476
1477 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1478         _, _, err := s.stdoutErrorRunHelper(c, `{
1479     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1480     "output_path": "/tmp",
1481     "state": "Locked"
1482 }`, func() {})
1483         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1484 }
1485
1486 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1487         _, _, err := s.stdoutErrorRunHelper(c, `{
1488     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1489     "output_path": "/tmp",
1490     "state": "Locked"
1491 }`, func() {})
1492         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1493 }
1494
1495 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1496         s.fullRunHelper(c, `{
1497     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1498     "container_image": "`+arvadostest.DockerImage112PDH+`",
1499     "cwd": "/bin",
1500     "environment": {},
1501     "mounts": {"/tmp": {"kind": "tmp"} },
1502     "output_path": "/tmp",
1503     "priority": 1,
1504     "runtime_constraints": {"API": true},
1505     "state": "Locked"
1506 }`, nil, 0, func() {
1507                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1508                 s.executor.exit <- 3
1509         })
1510         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1511         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1512 }
1513
1514 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1515         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1516         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1517         s.fullRunHelper(c, `{
1518     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1519     "container_image": "`+arvadostest.DockerImage112PDH+`",
1520     "cwd": "/bin",
1521     "environment": {},
1522     "mounts": {"/tmp": {"kind": "tmp"} },
1523     "output_path": "/tmp",
1524     "priority": 1,
1525     "runtime_constraints": {"API": true},
1526     "state": "Locked"
1527 }`, nil, 0, func() {
1528                 s.api.Container.Output = arvadostest.DockerImage112PDH
1529         })
1530
1531         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1532         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1533         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1534 }
1535
1536 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1537         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1538                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1539                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1540                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1541         }
1542         s.executor.runFunc = func() {
1543                 time.Sleep(time.Second)
1544                 s.executor.exit <- 0
1545         }
1546         record := `{
1547     "command": ["sleep", "1"],
1548     "container_image": "` + arvadostest.DockerImage112PDH + `",
1549     "cwd": "/bin",
1550     "environment": {},
1551     "mounts": {"/tmp": {"kind": "tmp"} },
1552     "output_path": "/tmp",
1553     "priority": 1,
1554     "runtime_constraints": {"API": true},
1555     "state": "Locked"
1556 }`
1557         err := json.Unmarshal([]byte(record), &s.api.Container)
1558         c.Assert(err, IsNil)
1559         err = s.runner.Run()
1560         c.Assert(err, IsNil)
1561         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1562         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1563         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1564         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1565 }
1566
1567 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1568         helperRecord := `{
1569                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1570                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1571                 "cwd": "/bin",
1572                 "environment": {"FROBIZ": "bilbo"},
1573                 "mounts": {
1574         "/tmp": {"kind": "tmp"},
1575         "/tmp/foo": {"kind": "collection",
1576                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1577                      "exclude_from_output": true
1578         },
1579         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1580     },
1581                 "output_path": "/tmp",
1582                 "priority": 1,
1583                 "runtime_constraints": {},
1584                 "state": "Locked"
1585         }`
1586
1587         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1588
1589         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1590                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1591         })
1592
1593         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1594         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1595         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1596 }
1597
1598 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1599         helperRecord := `{
1600                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1601                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1602                 "cwd": "/bin",
1603                 "environment": {"FROBIZ": "bilbo"},
1604                 "mounts": {
1605         "/tmp": {"kind": "tmp"},
1606         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1607         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1608         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1609         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1610         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1611     },
1612                 "output_path": "/tmp",
1613                 "priority": 1,
1614                 "runtime_constraints": {},
1615                 "state": "Locked"
1616         }`
1617
1618         extraMounts := []string{
1619                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1620                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1621                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1622         }
1623
1624         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1625                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1626         })
1627
1628         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1629                 "/tmp":                   {realtemp + "/tmp1", false},
1630                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1631                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1632                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1633                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1634         })
1635
1636         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1637         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1638         for _, v := range api.Content {
1639                 if v["collection"] != nil {
1640                         c.Check(v["ensure_unique_name"], Equals, true)
1641                         collection := v["collection"].(arvadosclient.Dict)
1642                         if strings.Index(collection["name"].(string), "output") == 0 {
1643                                 manifest := collection["manifest_text"].(string)
1644
1645                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1646 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1647 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1648 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1649 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1650 `)
1651                         }
1652                 }
1653         }
1654 }
1655
1656 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1657         helperRecord := `{
1658                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1659                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1660                 "cwd": "/bin",
1661                 "environment": {"FROBIZ": "bilbo"},
1662                 "mounts": {
1663         "/tmp": {"kind": "tmp"},
1664         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1665         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1666     },
1667                 "output_path": "/tmp",
1668                 "priority": 1,
1669                 "runtime_constraints": {},
1670                 "state": "Locked"
1671         }`
1672
1673         extraMounts := []string{
1674                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1675         }
1676
1677         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1678                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1679         })
1680
1681         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1682         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1683         for _, v := range s.api.Content {
1684                 if v["collection"] != nil {
1685                         collection := v["collection"].(arvadosclient.Dict)
1686                         if strings.Index(collection["name"].(string), "output") == 0 {
1687                                 manifest := collection["manifest_text"].(string)
1688
1689                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1690 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1691 `)
1692                         }
1693                 }
1694         }
1695 }
1696
1697 func (s *TestSuite) TestOutputError(c *C) {
1698         helperRecord := `{
1699                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1700                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1701                 "cwd": "/bin",
1702                 "environment": {"FROBIZ": "bilbo"},
1703                 "mounts": {
1704                         "/tmp": {"kind": "tmp"}
1705                 },
1706                 "output_path": "/tmp",
1707                 "priority": 1,
1708                 "runtime_constraints": {},
1709                 "state": "Locked"
1710         }`
1711         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1712                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1713         })
1714
1715         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1716 }
1717
1718 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1719         helperRecord := `{
1720                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1721                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1722                 "cwd": "/bin",
1723                 "environment": {"FROBIZ": "bilbo"},
1724                 "mounts": {
1725         "/tmp": {"kind": "tmp"},
1726         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1727         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1728     },
1729                 "output_path": "/tmp",
1730                 "priority": 1,
1731                 "runtime_constraints": {},
1732                 "state": "Locked"
1733         }`
1734
1735         extraMounts := []string{
1736                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1737         }
1738
1739         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1740                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1741         })
1742
1743         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1744         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1745         for _, v := range api.Content {
1746                 if v["collection"] != nil {
1747                         collection := v["collection"].(arvadosclient.Dict)
1748                         if strings.Index(collection["name"].(string), "output") == 0 {
1749                                 manifest := collection["manifest_text"].(string)
1750                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1751 `)
1752                         }
1753                 }
1754         }
1755 }
1756
1757 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1758         helperRecord := `{
1759                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1760                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1761                 "cwd": "/bin",
1762                 "environment": {"FROBIZ": "bilbo"},
1763                 "mounts": {
1764         "/tmp": {"kind": "tmp"},
1765         "stdin": {"kind": "json", "content": "foo"},
1766         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1767     },
1768                 "output_path": "/tmp",
1769                 "priority": 1,
1770                 "runtime_constraints": {},
1771                 "state": "Locked"
1772         }`
1773
1774         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1775                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1776         })
1777
1778         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1779         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1780         for _, v := range api.Content {
1781                 if v["collection"] != nil {
1782                         collection := v["collection"].(arvadosclient.Dict)
1783                         if strings.Index(collection["name"].(string), "output") == 0 {
1784                                 manifest := collection["manifest_text"].(string)
1785                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1786 `)
1787                         }
1788                 }
1789         }
1790 }
1791
1792 func (s *TestSuite) TestStderrMount(c *C) {
1793         api, cr, _ := s.fullRunHelper(c, `{
1794     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1795     "container_image": "`+arvadostest.DockerImage112PDH+`",
1796     "cwd": ".",
1797     "environment": {},
1798     "mounts": {"/tmp": {"kind": "tmp"},
1799                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1800                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1801     "output_path": "/tmp",
1802     "priority": 1,
1803     "runtime_constraints": {},
1804     "state": "Locked"
1805 }`, nil, 1, func() {
1806                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1807                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1808         })
1809
1810         final := api.CalledWith("container.state", "Complete")
1811         c.Assert(final, NotNil)
1812         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1813         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1814
1815         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1816 }
1817
1818 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1819         s.api.callraw = true
1820         err := s.runner.fetchContainerRecord()
1821         c.Assert(err, IsNil)
1822         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1823         c.Logf("%#v", s.runner.Container)
1824         c.Check(err, IsNil)
1825         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1826 }
1827
1828 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1829         nextState := ""
1830         for _, setup := range []func(){
1831                 func() {
1832                         c.Log("// waitErr = ocl runtime error")
1833                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1834                         nextState = "Cancelled"
1835                 },
1836                 func() {
1837                         c.Log("// loadErr = cannot connect")
1838                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1839                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1840                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1841                         c.Assert(err, IsNil)
1842                         nextState = "Queued"
1843                 },
1844         } {
1845                 s.SetUpTest(c)
1846                 setup()
1847                 s.fullRunHelper(c, `{
1848     "command": ["echo", "hello world"],
1849     "container_image": "`+arvadostest.DockerImage112PDH+`",
1850     "cwd": ".",
1851     "environment": {},
1852     "mounts": {"/tmp": {"kind": "tmp"} },
1853     "output_path": "/tmp",
1854     "priority": 1,
1855     "runtime_constraints": {},
1856     "state": "Locked"
1857 }`, nil, 0, func() {})
1858                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1859                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1860                 if *brokenNodeHook != "" {
1861                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1862                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1863                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1864                 } else {
1865                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1866                 }
1867         }
1868 }
1869
1870 func (s *TestSuite) TestBadCommand(c *C) {
1871         for _, startError := range []string{
1872                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1873                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1874                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1875         } {
1876                 s.SetUpTest(c)
1877                 s.executor.startErr = errors.New(startError)
1878                 s.fullRunHelper(c, `{
1879     "command": ["echo", "hello world"],
1880     "container_image": "`+arvadostest.DockerImage112PDH+`",
1881     "cwd": ".",
1882     "environment": {},
1883     "mounts": {"/tmp": {"kind": "tmp"} },
1884     "output_path": "/tmp",
1885     "priority": 1,
1886     "runtime_constraints": {},
1887     "state": "Locked"
1888 }`, nil, 0, func() {})
1889                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1890                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1891         }
1892 }
1893
1894 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1895         helperRecord := `{
1896                 "command": ["true"],
1897                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1898                 "cwd": "/bin",
1899                 "mounts": {
1900                     "/tmp": {"kind": "tmp"},
1901                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1902                 },
1903                 "secret_mounts": {
1904                 },
1905                 "output_path": "/tmp",
1906                 "priority": 1,
1907                 "runtime_constraints": {},
1908                 "state": "Locked"
1909         }`
1910
1911         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1912                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1913                 c.Check(err, IsNil)
1914                 c.Check(string(content), Equals, "mypassword")
1915         })
1916
1917         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1918         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1919         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1920         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1921
1922         // under secret mounts, not captured in output
1923         helperRecord = `{
1924                 "command": ["true"],
1925                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1926                 "cwd": "/bin",
1927                 "mounts": {
1928                     "/tmp": {"kind": "tmp"}
1929                 },
1930                 "secret_mounts": {
1931                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1932                 },
1933                 "output_path": "/tmp",
1934                 "priority": 1,
1935                 "runtime_constraints": {},
1936                 "state": "Locked"
1937         }`
1938
1939         s.SetUpTest(c)
1940         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1941                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1942                 c.Check(err, IsNil)
1943                 c.Check(string(content), Equals, "mypassword")
1944         })
1945
1946         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1947         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1948         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
1949         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
1950 }
1951
1952 type FakeProcess struct {
1953         cmdLine []string
1954 }
1955
1956 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
1957         return fp.cmdLine, nil
1958 }