services/crunch-run/crunchrun_test.go

   1 package main
   2
   3 import (
   4         "bytes"
   5         "crypto/md5"
   6         "encoding/json"
   7         "errors"
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "os"
  12         "os/exec"
  13         "path/filepath"
  14         "runtime/pprof"
  15         "sort"
  16         "strings"
  17         "sync"
  18         "syscall"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvados"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         "git.curoverse.com/arvados.git/sdk/go/manifest"
  26         "github.com/curoverse/dockerclient"
  27         . "gopkg.in/check.v1"
  28 )
  29
  30 // Gocheck boilerplate
  31 func TestCrunchExec(t *testing.T) {
  32         TestingT(t)
  33 }
  34
  35 type TestSuite struct{}
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&TestSuite{})
  39
  40 type ArvTestClient struct {
  41         Total   int64
  42         Calls   int
  43         Content []arvadosclient.Dict
  44         arvados.Container
  45         Logs map[string]*bytes.Buffer
  46         sync.Mutex
  47         WasSetRunning bool
  48 }
  49
  50 type KeepTestClient struct {
  51         Called  bool
  52         Content []byte
  53 }
  54
  55 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
  56 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
  57 var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
  58
  59 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
  60 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
  61
  62 var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
  63 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
  64
  65 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
  66 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
  67
  68 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
  69 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
  70
  71 type TestDockerClient struct {
  72         imageLoaded string
  73         logReader   io.ReadCloser
  74         logWriter   io.WriteCloser
  75         fn          func(t *TestDockerClient)
  76         finish      chan dockerclient.WaitResult
  77         stop        chan bool
  78         cwd         string
  79         env         []string
  80         api         *ArvTestClient
  81 }
  82
  83 func NewTestDockerClient() *TestDockerClient {
  84         t := &TestDockerClient{}
  85         t.logReader, t.logWriter = io.Pipe()
  86         t.finish = make(chan dockerclient.WaitResult)
  87         t.stop = make(chan bool)
  88         t.cwd = "/"
  89         return t
  90 }
  91
  92 func (t *TestDockerClient) StopContainer(id string, timeout int) error {
  93         t.stop <- true
  94         return nil
  95 }
  96
  97 func (t *TestDockerClient) InspectImage(id string) (*dockerclient.ImageInfo, error) {
  98         if t.imageLoaded == id {
  99                 return &dockerclient.ImageInfo{}, nil
 100         } else {
 101                 return nil, errors.New("")
 102         }
 103 }
 104
 105 func (t *TestDockerClient) LoadImage(reader io.Reader) error {
 106         _, err := io.Copy(ioutil.Discard, reader)
 107         if err != nil {
 108                 return err
 109         } else {
 110                 t.imageLoaded = hwImageId
 111                 return nil
 112         }
 113 }
 114
 115 func (t *TestDockerClient) CreateContainer(config *dockerclient.ContainerConfig, name string, authConfig *dockerclient.AuthConfig) (string, error) {
 116         if config.WorkingDir != "" {
 117                 t.cwd = config.WorkingDir
 118         }
 119         t.env = config.Env
 120         return "abcde", nil
 121 }
 122
 123 func (t *TestDockerClient) StartContainer(id string, config *dockerclient.HostConfig) error {
 124         if id == "abcde" {
 125                 go t.fn(t)
 126                 return nil
 127         } else {
 128                 return errors.New("Invalid container id")
 129         }
 130 }
 131
 132 func (t *TestDockerClient) AttachContainer(id string, options *dockerclient.AttachOptions) (io.ReadCloser, error) {
 133         return t.logReader, nil
 134 }
 135
 136 func (t *TestDockerClient) Wait(id string) <-chan dockerclient.WaitResult {
 137         return t.finish
 138 }
 139
 140 func (*TestDockerClient) RemoveImage(name string, force bool) ([]*dockerclient.ImageDelete, error) {
 141         return nil, nil
 142 }
 143
 144 func (client *ArvTestClient) Create(resourceType string,
 145         parameters arvadosclient.Dict,
 146         output interface{}) error {
 147
 148         client.Mutex.Lock()
 149         defer client.Mutex.Unlock()
 150
 151         client.Calls++
 152         client.Content = append(client.Content, parameters)
 153
 154         if resourceType == "logs" {
 155                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 156                 if client.Logs == nil {
 157                         client.Logs = make(map[string]*bytes.Buffer)
 158                 }
 159                 if client.Logs[et] == nil {
 160                         client.Logs[et] = &bytes.Buffer{}
 161                 }
 162                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 163         }
 164
 165         if resourceType == "collections" && output != nil {
 166                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 167                 outmap := output.(*arvados.Collection)
 168                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 169         }
 170
 171         return nil
 172 }
 173
 174 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 175         switch {
 176         case method == "GET" && resourceType == "containers" && action == "auth":
 177                 return json.Unmarshal([]byte(`{
 178                         "kind": "arvados#api_client_authorization",
 179                         "uuid": "`+fakeAuthUUID+`",
 180                         "api_token": "`+fakeAuthToken+`"
 181                         }`), output)
 182         default:
 183                 return fmt.Errorf("Not found")
 184         }
 185 }
 186
 187 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 188         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 189         j := []byte(`{
 190                 "command": ["sleep", "1"],
 191                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 192                 "cwd": ".",
 193                 "environment": {},
 194                 "mounts": {"/tmp": {"kind": "tmp"} },
 195                 "output_path": "/tmp",
 196                 "priority": 1,
 197                 "runtime_constraints": {}
 198         }`)
 199         return ioutil.NopCloser(bytes.NewReader(j)), nil
 200 }
 201
 202 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 203         if resourceType == "collections" {
 204                 if uuid == hwPDH {
 205                         output.(*arvados.Collection).ManifestText = hwManifest
 206                 } else if uuid == otherPDH {
 207                         output.(*arvados.Collection).ManifestText = otherManifest
 208                 } else if uuid == normalizedWithSubdirsPDH {
 209                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 210                 } else if uuid == denormalizedWithSubdirsPDH {
 211                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 212                 }
 213         }
 214         if resourceType == "containers" {
 215                 (*output.(*arvados.Container)) = client.Container
 216         }
 217         return nil
 218 }
 219
 220 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 221         client.Mutex.Lock()
 222         defer client.Mutex.Unlock()
 223         client.Calls++
 224         client.Content = append(client.Content, parameters)
 225         if resourceType == "containers" {
 226                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 227                         client.WasSetRunning = true
 228                 }
 229         }
 230         return nil
 231 }
 232
 233 var discoveryMap = map[string]interface{}{"defaultTrashLifetime": float64(1209600)}
 234
 235 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 236         return discoveryMap[key], nil
 237 }
 238
 239 // CalledWith returns the parameters from the first API call whose
 240 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 241 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 242 // no call matches, it returns nil.
 243 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 244 call:
 245         for _, content := range client.Content {
 246                 var v interface{} = content
 247                 for _, k := range strings.Split(jpath, ".") {
 248                         if dict, ok := v.(arvadosclient.Dict); !ok {
 249                                 continue call
 250                         } else {
 251                                 v = dict[k]
 252                         }
 253                 }
 254                 if v == expect {
 255                         return content
 256                 }
 257         }
 258         return nil
 259 }
 260
 261 func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 262         client.Content = buf
 263         return fmt.Sprintf("%s+%d", hash, len(buf)), len(buf), nil
 264 }
 265
 266 type FileWrapper struct {
 267         io.ReadCloser
 268         len uint64
 269 }
 270
 271 func (fw FileWrapper) Len() uint64 {
 272         return fw.len
 273 }
 274
 275 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 276         return 0, errors.New("not implemented")
 277 }
 278
 279 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 280         if filename == hwImageId+".tar" {
 281                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 282                 client.Called = true
 283                 return FileWrapper{rdr, 1321984}, nil
 284         }
 285         return nil, nil
 286 }
 287
 288 func (s *TestSuite) TestLoadImage(c *C) {
 289         kc := &KeepTestClient{}
 290         docker := NewTestDockerClient()
 291         cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 292
 293         _, err := cr.Docker.RemoveImage(hwImageId, true)
 294
 295         _, err = cr.Docker.InspectImage(hwImageId)
 296         c.Check(err, NotNil)
 297
 298         cr.Container.ContainerImage = hwPDH
 299
 300         // (1) Test loading image from keep
 301         c.Check(kc.Called, Equals, false)
 302         c.Check(cr.ContainerConfig.Image, Equals, "")
 303
 304         err = cr.LoadImage()
 305
 306         c.Check(err, IsNil)
 307         defer func() {
 308                 cr.Docker.RemoveImage(hwImageId, true)
 309         }()
 310
 311         c.Check(kc.Called, Equals, true)
 312         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 313
 314         _, err = cr.Docker.InspectImage(hwImageId)
 315         c.Check(err, IsNil)
 316
 317         // (2) Test using image that's already loaded
 318         kc.Called = false
 319         cr.ContainerConfig.Image = ""
 320
 321         err = cr.LoadImage()
 322         c.Check(err, IsNil)
 323         c.Check(kc.Called, Equals, false)
 324         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 325
 326 }
 327
 328 type ArvErrorTestClient struct{}
 329
 330 func (ArvErrorTestClient) Create(resourceType string,
 331         parameters arvadosclient.Dict,
 332         output interface{}) error {
 333         return nil
 334 }
 335
 336 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 337         return errors.New("ArvError")
 338 }
 339
 340 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 341         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 342         return nil, errors.New("ArvError")
 343 }
 344
 345 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 346         return errors.New("ArvError")
 347 }
 348
 349 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 350         return nil
 351 }
 352
 353 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 354         return discoveryMap[key], nil
 355 }
 356
 357 type KeepErrorTestClient struct{}
 358
 359 func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 360         return "", 0, errors.New("KeepError")
 361 }
 362
 363 func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 364         return nil, errors.New("KeepError")
 365 }
 366
 367 type KeepReadErrorTestClient struct{}
 368
 369 func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 370         return "", 0, nil
 371 }
 372
 373 type ErrorReader struct{}
 374
 375 func (ErrorReader) Read(p []byte) (n int, err error) {
 376         return 0, errors.New("ErrorReader")
 377 }
 378
 379 func (ErrorReader) Close() error {
 380         return nil
 381 }
 382
 383 func (ErrorReader) Len() uint64 {
 384         return 0
 385 }
 386
 387 func (ErrorReader) Seek(int64, int) (int64, error) {
 388         return 0, errors.New("ErrorReader")
 389 }
 390
 391 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 392         return ErrorReader{}, nil
 393 }
 394
 395 func (s *TestSuite) TestLoadImageArvError(c *C) {
 396         // (1) Arvados error
 397         cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 398         cr.Container.ContainerImage = hwPDH
 399
 400         err := cr.LoadImage()
 401         c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
 402 }
 403
 404 func (s *TestSuite) TestLoadImageKeepError(c *C) {
 405         // (2) Keep error
 406         docker := NewTestDockerClient()
 407         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 408         cr.Container.ContainerImage = hwPDH
 409
 410         err := cr.LoadImage()
 411         c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError")
 412 }
 413
 414 func (s *TestSuite) TestLoadImageCollectionError(c *C) {
 415         // (3) Collection doesn't contain image
 416         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 417         cr.Container.ContainerImage = otherPDH
 418
 419         err := cr.LoadImage()
 420         c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
 421 }
 422
 423 func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
 424         // (4) Collection doesn't contain image
 425         docker := NewTestDockerClient()
 426         cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 427         cr.Container.ContainerImage = hwPDH
 428
 429         err := cr.LoadImage()
 430         c.Check(err, NotNil)
 431 }
 432
 433 type ClosableBuffer struct {
 434         bytes.Buffer
 435 }
 436
 437 func (*ClosableBuffer) Close() error {
 438         return nil
 439 }
 440
 441 type TestLogs struct {
 442         Stdout ClosableBuffer
 443         Stderr ClosableBuffer
 444 }
 445
 446 func (tl *TestLogs) NewTestLoggingWriter(logstr string) io.WriteCloser {
 447         if logstr == "stdout" {
 448                 return &tl.Stdout
 449         }
 450         if logstr == "stderr" {
 451                 return &tl.Stderr
 452         }
 453         return nil
 454 }
 455
 456 func dockerLog(fd byte, msg string) []byte {
 457         by := []byte(msg)
 458         header := make([]byte, 8+len(by))
 459         header[0] = fd
 460         header[7] = byte(len(by))
 461         copy(header[8:], by)
 462         return header
 463 }
 464
 465 func (s *TestSuite) TestRunContainer(c *C) {
 466         docker := NewTestDockerClient()
 467         docker.fn = func(t *TestDockerClient) {
 468                 t.logWriter.Write(dockerLog(1, "Hello world\n"))
 469                 t.logWriter.Close()
 470                 t.finish <- dockerclient.WaitResult{}
 471         }
 472         cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 473
 474         var logs TestLogs
 475         cr.NewLogWriter = logs.NewTestLoggingWriter
 476         cr.Container.ContainerImage = hwPDH
 477         cr.Container.Command = []string{"./hw"}
 478         err := cr.LoadImage()
 479         c.Check(err, IsNil)
 480
 481         err = cr.CreateContainer()
 482         c.Check(err, IsNil)
 483
 484         err = cr.StartContainer()
 485         c.Check(err, IsNil)
 486
 487         err = cr.WaitFinish()
 488         c.Check(err, IsNil)
 489
 490         c.Check(strings.HasSuffix(logs.Stdout.String(), "Hello world\n"), Equals, true)
 491         c.Check(logs.Stderr.String(), Equals, "")
 492 }
 493
 494 func (s *TestSuite) TestCommitLogs(c *C) {
 495         api := &ArvTestClient{}
 496         kc := &KeepTestClient{}
 497         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 498         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 499
 500         cr.CrunchLog.Print("Hello world!")
 501         cr.CrunchLog.Print("Goodbye")
 502         cr.finalState = "Complete"
 503
 504         err := cr.CommitLogs()
 505         c.Check(err, IsNil)
 506
 507         c.Check(api.Calls, Equals, 2)
 508         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 509         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 510         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 511         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 512 }
 513
 514 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 515         api := &ArvTestClient{}
 516         kc := &KeepTestClient{}
 517         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 518
 519         err := cr.UpdateContainerRunning()
 520         c.Check(err, IsNil)
 521
 522         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 523 }
 524
 525 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 526         api := &ArvTestClient{}
 527         kc := &KeepTestClient{}
 528         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 529
 530         cr.LogsPDH = new(string)
 531         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 532
 533         cr.ExitCode = new(int)
 534         *cr.ExitCode = 42
 535         cr.finalState = "Complete"
 536
 537         err := cr.UpdateContainerFinal()
 538         c.Check(err, IsNil)
 539
 540         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 541         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 542         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 543 }
 544
 545 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 546         api := &ArvTestClient{}
 547         kc := &KeepTestClient{}
 548         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 549         cr.cCancelled = true
 550         cr.finalState = "Cancelled"
 551
 552         err := cr.UpdateContainerFinal()
 553         c.Check(err, IsNil)
 554
 555         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 556         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 557         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 558 }
 559
 560 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 561 // dress rehearsal of the Run() function, starting from a JSON container record.
 562 func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
 563         rec := arvados.Container{}
 564         err := json.Unmarshal([]byte(record), &rec)
 565         c.Check(err, IsNil)
 566
 567         docker := NewTestDockerClient()
 568         docker.fn = fn
 569         docker.RemoveImage(hwImageId, true)
 570
 571         api = &ArvTestClient{Container: rec}
 572         docker.api = api
 573         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 574         cr.statInterval = 100 * time.Millisecond
 575         am := &ArvMountCmdLine{}
 576         cr.RunArvMount = am.ArvMountTest
 577
 578         realTemp, err = ioutil.TempDir("", "crunchrun_test1-")
 579         c.Assert(err, IsNil)
 580         defer os.RemoveAll(realTemp)
 581
 582         tempcount := 0
 583         cr.MkTempDir = func(_ string, prefix string) (string, error) {
 584                 tempcount++
 585                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 586                 err := os.Mkdir(d, os.ModePerm)
 587                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 588                         // Test case must have pre-populated the tempdir
 589                         err = nil
 590                 }
 591                 return d, err
 592         }
 593
 594         if extraMounts != nil && len(extraMounts) > 0 {
 595                 err := cr.SetupArvMountPoint("keep")
 596                 c.Check(err, IsNil)
 597
 598                 for _, m := range extraMounts {
 599                         os.MkdirAll(cr.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 600                 }
 601         }
 602
 603         err = cr.Run()
 604         c.Check(err, IsNil)
 605         c.Check(api.WasSetRunning, Equals, true)
 606
 607         c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil)
 608
 609         if err != nil {
 610                 for k, v := range api.Logs {
 611                         c.Log(k)
 612                         c.Log(v.String())
 613                 }
 614         }
 615
 616         return
 617 }
 618
 619 func (s *TestSuite) TestFullRunHello(c *C) {
 620         api, _, _ := FullRunHelper(c, `{
 621     "command": ["echo", "hello world"],
 622     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 623     "cwd": ".",
 624     "environment": {},
 625     "mounts": {"/tmp": {"kind": "tmp"} },
 626     "output_path": "/tmp",
 627     "priority": 1,
 628     "runtime_constraints": {}
 629 }`, nil, func(t *TestDockerClient) {
 630                 t.logWriter.Write(dockerLog(1, "hello world\n"))
 631                 t.logWriter.Close()
 632                 t.finish <- dockerclient.WaitResult{}
 633         })
 634
 635         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 636         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 637         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello world\n"), Equals, true)
 638
 639 }
 640
 641 func (s *TestSuite) TestCrunchstat(c *C) {
 642         api, _, _ := FullRunHelper(c, `{
 643                 "command": ["sleep", "1"],
 644                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 645                 "cwd": ".",
 646                 "environment": {},
 647                 "mounts": {"/tmp": {"kind": "tmp"} },
 648                 "output_path": "/tmp",
 649                 "priority": 1,
 650                 "runtime_constraints": {}
 651         }`, nil, func(t *TestDockerClient) {
 652                 time.Sleep(time.Second)
 653                 t.logWriter.Close()
 654                 t.finish <- dockerclient.WaitResult{}
 655         })
 656
 657         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 658         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 659
 660         // We didn't actually start a container, so crunchstat didn't
 661         // find accounting files and therefore didn't log any stats.
 662         // It should have logged a "can't find accounting files"
 663         // message after one poll interval, though, so we can confirm
 664         // it's alive:
 665         c.Assert(api.Logs["crunchstat"], NotNil)
 666         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 667
 668         // The "files never appeared" log assures us that we called
 669         // (*crunchstat.Reporter)Stop(), and that we set it up with
 670         // the correct container ID "abcde":
 671         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
 672 }
 673
 674 func (s *TestSuite) TestNodeInfoLog(c *C) {
 675         api, _, _ := FullRunHelper(c, `{
 676                 "command": ["sleep", "1"],
 677                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 678                 "cwd": ".",
 679                 "environment": {},
 680                 "mounts": {"/tmp": {"kind": "tmp"} },
 681                 "output_path": "/tmp",
 682                 "priority": 1,
 683                 "runtime_constraints": {}
 684         }`, nil, func(t *TestDockerClient) {
 685                 time.Sleep(time.Second)
 686                 t.logWriter.Close()
 687                 t.finish <- dockerclient.WaitResult{}
 688         })
 689
 690         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 691         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 692
 693         c.Assert(api.Logs["node-info"], NotNil)
 694         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
 695         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
 696         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
 697         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
 698         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`)
 699 }
 700
 701 func (s *TestSuite) TestContainerRecordLog(c *C) {
 702         api, _, _ := FullRunHelper(c, `{
 703                 "command": ["sleep", "1"],
 704                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 705                 "cwd": ".",
 706                 "environment": {},
 707                 "mounts": {"/tmp": {"kind": "tmp"} },
 708                 "output_path": "/tmp",
 709                 "priority": 1,
 710                 "runtime_constraints": {}
 711         }`, nil, func(t *TestDockerClient) {
 712                 time.Sleep(time.Second)
 713                 t.logWriter.Close()
 714                 t.finish <- dockerclient.WaitResult{}
 715         })
 716
 717         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 718         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 719
 720         c.Assert(api.Logs["container"], NotNil)
 721         c.Check(api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 722 }
 723
 724 func (s *TestSuite) TestFullRunStderr(c *C) {
 725         api, _, _ := FullRunHelper(c, `{
 726     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 727     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 728     "cwd": ".",
 729     "environment": {},
 730     "mounts": {"/tmp": {"kind": "tmp"} },
 731     "output_path": "/tmp",
 732     "priority": 1,
 733     "runtime_constraints": {}
 734 }`, nil, func(t *TestDockerClient) {
 735                 t.logWriter.Write(dockerLog(1, "hello\n"))
 736                 t.logWriter.Write(dockerLog(2, "world\n"))
 737                 t.logWriter.Close()
 738                 t.finish <- dockerclient.WaitResult{ExitCode: 1}
 739         })
 740
 741         final := api.CalledWith("container.state", "Complete")
 742         c.Assert(final, NotNil)
 743         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 744         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 745
 746         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello\n"), Equals, true)
 747         c.Check(strings.HasSuffix(api.Logs["stderr"].String(), "world\n"), Equals, true)
 748 }
 749
 750 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 751         api, _, _ := FullRunHelper(c, `{
 752     "command": ["pwd"],
 753     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 754     "cwd": ".",
 755     "environment": {},
 756     "mounts": {"/tmp": {"kind": "tmp"} },
 757     "output_path": "/tmp",
 758     "priority": 1,
 759     "runtime_constraints": {}
 760 }`, nil, func(t *TestDockerClient) {
 761                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 762                 t.logWriter.Close()
 763                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 764         })
 765
 766         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 767         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 768         c.Log(api.Logs["stdout"])
 769         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/\n"), Equals, true)
 770 }
 771
 772 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 773         api, _, _ := FullRunHelper(c, `{
 774     "command": ["pwd"],
 775     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 776     "cwd": "/bin",
 777     "environment": {},
 778     "mounts": {"/tmp": {"kind": "tmp"} },
 779     "output_path": "/tmp",
 780     "priority": 1,
 781     "runtime_constraints": {}
 782 }`, nil, func(t *TestDockerClient) {
 783                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 784                 t.logWriter.Close()
 785                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 786         })
 787
 788         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 789         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 790         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/bin\n"), Equals, true)
 791 }
 792
 793 func (s *TestSuite) TestStopOnSignal(c *C) {
 794         s.testStopContainer(c, func(cr *ContainerRunner) {
 795                 go func() {
 796                         for !cr.cStarted {
 797                                 time.Sleep(time.Millisecond)
 798                         }
 799                         cr.SigChan <- syscall.SIGINT
 800                 }()
 801         })
 802 }
 803
 804 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 805         s.testStopContainer(c, func(cr *ContainerRunner) {
 806                 cr.ArvMountExit = make(chan error)
 807                 go func() {
 808                         cr.ArvMountExit <- exec.Command("true").Run()
 809                         close(cr.ArvMountExit)
 810                 }()
 811         })
 812 }
 813
 814 func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
 815         record := `{
 816     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 817     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 818     "cwd": ".",
 819     "environment": {},
 820     "mounts": {"/tmp": {"kind": "tmp"} },
 821     "output_path": "/tmp",
 822     "priority": 1,
 823     "runtime_constraints": {}
 824 }`
 825
 826         rec := arvados.Container{}
 827         err := json.Unmarshal([]byte(record), &rec)
 828         c.Check(err, IsNil)
 829
 830         docker := NewTestDockerClient()
 831         docker.fn = func(t *TestDockerClient) {
 832                 <-t.stop
 833                 t.logWriter.Write(dockerLog(1, "foo\n"))
 834                 t.logWriter.Close()
 835                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 836         }
 837         docker.RemoveImage(hwImageId, true)
 838
 839         api := &ArvTestClient{Container: rec}
 840         cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 841         cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 842         setup(cr)
 843
 844         done := make(chan error)
 845         go func() {
 846                 done <- cr.Run()
 847         }()
 848         select {
 849         case <-time.After(20 * time.Second):
 850                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
 851                 c.Fatal("timed out")
 852         case err = <-done:
 853                 c.Check(err, IsNil)
 854         }
 855         for k, v := range api.Logs {
 856                 c.Log(k)
 857                 c.Log(v.String())
 858         }
 859
 860         c.Check(api.CalledWith("container.log", nil), NotNil)
 861         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
 862         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "foo\n"), Equals, true)
 863 }
 864
 865 func (s *TestSuite) TestFullRunSetEnv(c *C) {
 866         api, _, _ := FullRunHelper(c, `{
 867     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
 868     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 869     "cwd": "/bin",
 870     "environment": {"FROBIZ": "bilbo"},
 871     "mounts": {"/tmp": {"kind": "tmp"} },
 872     "output_path": "/tmp",
 873     "priority": 1,
 874     "runtime_constraints": {}
 875 }`, nil, func(t *TestDockerClient) {
 876                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 877                 t.logWriter.Close()
 878                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 879         })
 880
 881         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 882         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 883         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "bilbo\n"), Equals, true)
 884 }
 885
 886 type ArvMountCmdLine struct {
 887         Cmd   []string
 888         token string
 889 }
 890
 891 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
 892         am.Cmd = c
 893         am.token = token
 894         return nil, nil
 895 }
 896
 897 func stubCert(temp string) string {
 898         path := temp + "/ca-certificates.crt"
 899         crt, _ := os.Create(path)
 900         crt.Close()
 901         arvadosclient.CertFiles = []string{path}
 902         return path
 903 }
 904
 905 func (s *TestSuite) TestSetupMounts(c *C) {
 906         api := &ArvTestClient{}
 907         kc := &KeepTestClient{}
 908         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 909         am := &ArvMountCmdLine{}
 910         cr.RunArvMount = am.ArvMountTest
 911
 912         realTemp, err := ioutil.TempDir("", "crunchrun_test1-")
 913         c.Assert(err, IsNil)
 914         certTemp, err := ioutil.TempDir("", "crunchrun_test2-")
 915         c.Assert(err, IsNil)
 916         stubCertPath := stubCert(certTemp)
 917
 918         defer os.RemoveAll(realTemp)
 919         defer os.RemoveAll(certTemp)
 920
 921         i := 0
 922         cr.MkTempDir = func(_ string, prefix string) (string, error) {
 923                 i++
 924                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
 925                 err := os.Mkdir(d, os.ModePerm)
 926                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 927                         // Test case must have pre-populated the tempdir
 928                         err = nil
 929                 }
 930                 return d, err
 931         }
 932
 933         checkEmpty := func() {
 934                 filepath.Walk(realTemp, func(path string, _ os.FileInfo, err error) error {
 935                         c.Check(path, Equals, realTemp)
 936                         c.Check(err, IsNil)
 937                         return nil
 938                 })
 939         }
 940
 941         {
 942                 i = 0
 943                 cr.ArvMountPoint = ""
 944                 cr.Container.Mounts = make(map[string]arvados.Mount)
 945                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
 946                 cr.OutputPath = "/tmp"
 947
 948                 err := cr.SetupMounts()
 949                 c.Check(err, IsNil)
 950                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 951                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp"})
 952                 cr.CleanupDirs()
 953                 checkEmpty()
 954         }
 955
 956         {
 957                 i = 0
 958                 cr.ArvMountPoint = ""
 959                 cr.Container.Mounts = make(map[string]arvados.Mount)
 960                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
 961                 cr.OutputPath = "/tmp"
 962
 963                 apiflag := true
 964                 cr.Container.RuntimeConstraints.API = &apiflag
 965
 966                 err := cr.SetupMounts()
 967                 c.Check(err, IsNil)
 968                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 969                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"})
 970                 cr.CleanupDirs()
 971                 checkEmpty()
 972
 973                 apiflag = false
 974         }
 975
 976         {
 977                 i = 0
 978                 cr.ArvMountPoint = ""
 979                 cr.Container.Mounts = map[string]arvados.Mount{
 980                         "/keeptmp": {Kind: "collection", Writable: true},
 981                 }
 982                 cr.OutputPath = "/keeptmp"
 983
 984                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
 985
 986                 err := cr.SetupMounts()
 987                 c.Check(err, IsNil)
 988                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 989                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/tmp0:/keeptmp"})
 990                 cr.CleanupDirs()
 991                 checkEmpty()
 992         }
 993
 994         {
 995                 i = 0
 996                 cr.ArvMountPoint = ""
 997                 cr.Container.Mounts = map[string]arvados.Mount{
 998                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
 999                         "/keepout": {Kind: "collection", Writable: true},
1000                 }
1001                 cr.OutputPath = "/keepout"
1002
1003                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1004                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1005
1006                 err := cr.SetupMounts()
1007                 c.Check(err, IsNil)
1008                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1009                 sort.StringSlice(cr.Binds).Sort()
1010                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
1011                         realTemp + "/keep1/tmp0:/keepout"})
1012                 cr.CleanupDirs()
1013                 checkEmpty()
1014         }
1015
1016         {
1017                 i = 0
1018                 cr.ArvMountPoint = ""
1019                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1020                 cr.Container.Mounts = map[string]arvados.Mount{
1021                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1022                         "/keepout": {Kind: "collection", Writable: true},
1023                 }
1024                 cr.OutputPath = "/keepout"
1025
1026                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1027                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1028
1029                 err := cr.SetupMounts()
1030                 c.Check(err, IsNil)
1031                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1032                 sort.StringSlice(cr.Binds).Sort()
1033                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
1034                         realTemp + "/keep1/tmp0:/keepout"})
1035                 cr.CleanupDirs()
1036                 checkEmpty()
1037         }
1038
1039         for _, test := range []struct {
1040                 in  interface{}
1041                 out string
1042         }{
1043                 {in: "foo", out: `"foo"`},
1044                 {in: nil, out: `null`},
1045                 {in: map[string]int{"foo": 123}, out: `{"foo":123}`},
1046         } {
1047                 i = 0
1048                 cr.ArvMountPoint = ""
1049                 cr.Container.Mounts = map[string]arvados.Mount{
1050                         "/mnt/test.json": {Kind: "json", Content: test.in},
1051                 }
1052                 err := cr.SetupMounts()
1053                 c.Check(err, IsNil)
1054                 sort.StringSlice(cr.Binds).Sort()
1055                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2/mountdata.json:/mnt/test.json:ro"})
1056                 content, err := ioutil.ReadFile(realTemp + "/2/mountdata.json")
1057                 c.Check(err, IsNil)
1058                 c.Check(content, DeepEquals, []byte(test.out))
1059                 cr.CleanupDirs()
1060                 checkEmpty()
1061         }
1062
1063         // Read-only mount points are allowed underneath output_dir mount point
1064         {
1065                 i = 0
1066                 cr.ArvMountPoint = ""
1067                 cr.Container.Mounts = make(map[string]arvados.Mount)
1068                 cr.Container.Mounts = map[string]arvados.Mount{
1069                         "/tmp":     {Kind: "tmp"},
1070                         "/tmp/foo": {Kind: "collection"},
1071                 }
1072                 cr.OutputPath = "/tmp"
1073
1074                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1075
1076                 err := cr.SetupMounts()
1077                 c.Check(err, IsNil)
1078                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1079                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"})
1080                 cr.CleanupDirs()
1081                 checkEmpty()
1082         }
1083
1084         // Writable mount points are not allowed underneath output_dir mount point
1085         {
1086                 i = 0
1087                 cr.ArvMountPoint = ""
1088                 cr.Container.Mounts = make(map[string]arvados.Mount)
1089                 cr.Container.Mounts = map[string]arvados.Mount{
1090                         "/tmp":     {Kind: "tmp"},
1091                         "/tmp/foo": {Kind: "collection", Writable: true},
1092                 }
1093                 cr.OutputPath = "/tmp"
1094
1095                 err := cr.SetupMounts()
1096                 c.Check(err, NotNil)
1097                 c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
1098                 cr.CleanupDirs()
1099                 checkEmpty()
1100         }
1101
1102         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1103         {
1104                 i = 0
1105                 cr.ArvMountPoint = ""
1106                 cr.Container.Mounts = make(map[string]arvados.Mount)
1107                 cr.Container.Mounts = map[string]arvados.Mount{
1108                         "/tmp":     {Kind: "tmp"},
1109                         "/tmp/foo": {Kind: "json"},
1110                 }
1111                 cr.OutputPath = "/tmp"
1112
1113                 err := cr.SetupMounts()
1114                 c.Check(err, NotNil)
1115                 c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`)
1116                 cr.CleanupDirs()
1117                 checkEmpty()
1118         }
1119 }
1120
1121 func (s *TestSuite) TestStdout(c *C) {
1122         helperRecord := `{
1123                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1124                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1125                 "cwd": "/bin",
1126                 "environment": {"FROBIZ": "bilbo"},
1127                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1128                 "output_path": "/tmp",
1129                 "priority": 1,
1130                 "runtime_constraints": {}
1131         }`
1132
1133         api, _, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
1134                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1135                 t.logWriter.Close()
1136                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1137         })
1138
1139         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1140         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1141         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1142 }
1143
1144 // Used by the TestStdoutWithWrongPath*()
1145 func StdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, err error) {
1146         rec := arvados.Container{}
1147         err = json.Unmarshal([]byte(record), &rec)
1148         c.Check(err, IsNil)
1149
1150         docker := NewTestDockerClient()
1151         docker.fn = fn
1152         docker.RemoveImage(hwImageId, true)
1153
1154         api = &ArvTestClient{Container: rec}
1155         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
1156         am := &ArvMountCmdLine{}
1157         cr.RunArvMount = am.ArvMountTest
1158
1159         err = cr.Run()
1160         return
1161 }
1162
1163 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1164         _, _, err := StdoutErrorRunHelper(c, `{
1165     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1166     "output_path": "/tmp"
1167 }`, func(t *TestDockerClient) {})
1168
1169         c.Check(err, NotNil)
1170         c.Check(strings.Contains(err.Error(), "Stdout path does not start with OutputPath"), Equals, true)
1171 }
1172
1173 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1174         _, _, err := StdoutErrorRunHelper(c, `{
1175     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1176     "output_path": "/tmp"
1177 }`, func(t *TestDockerClient) {})
1178
1179         c.Check(err, NotNil)
1180         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
1181 }
1182
1183 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1184         _, _, err := StdoutErrorRunHelper(c, `{
1185     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1186     "output_path": "/tmp"
1187 }`, func(t *TestDockerClient) {})
1188
1189         c.Check(err, NotNil)
1190         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
1191 }
1192
1193 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1194         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1195         defer os.Unsetenv("ARVADOS_API_HOST")
1196         api, _, _ := FullRunHelper(c, `{
1197     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1198     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1199     "cwd": "/bin",
1200     "environment": {},
1201     "mounts": {"/tmp": {"kind": "tmp"} },
1202     "output_path": "/tmp",
1203     "priority": 1,
1204     "runtime_constraints": {"API": true}
1205 }`, nil, func(t *TestDockerClient) {
1206                 t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
1207                 t.logWriter.Close()
1208                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1209         })
1210
1211         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1212         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1213         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "test.arvados.org\n"), Equals, true)
1214         c.Check(api.CalledWith("container.output", "d41d8cd98f00b204e9800998ecf8427e+0"), NotNil)
1215 }
1216
1217 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1218         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1219         defer os.Unsetenv("ARVADOS_API_HOST")
1220         api, _, _ := FullRunHelper(c, `{
1221     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1222     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1223     "cwd": "/bin",
1224     "environment": {},
1225     "mounts": {"/tmp": {"kind": "tmp"} },
1226     "output_path": "/tmp",
1227     "priority": 1,
1228     "runtime_constraints": {"API": true}
1229 }`, nil, func(t *TestDockerClient) {
1230                 t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
1231                 t.logWriter.Close()
1232                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1233         })
1234
1235         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1236         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1237         c.Check(api.CalledWith("container.output", "d4ab34d3d4f8a72f5c4973051ae69fab+122"), NotNil)
1238 }
1239
1240 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1241         helperRecord := `{
1242                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1243                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1244                 "cwd": "/bin",
1245                 "environment": {"FROBIZ": "bilbo"},
1246                 "mounts": {
1247         "/tmp": {"kind": "tmp"},
1248         "/tmp/foo": {"kind": "collection",
1249                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1250                      "exclude_from_output": true
1251         },
1252         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1253     },
1254                 "output_path": "/tmp",
1255                 "priority": 1,
1256                 "runtime_constraints": {}
1257         }`
1258
1259         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1260
1261         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
1262                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1263                 t.logWriter.Close()
1264                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1265         })
1266
1267         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1268         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1269         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1270 }
1271
1272 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1273         helperRecord := `{
1274                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1275                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1276                 "cwd": "/bin",
1277                 "environment": {"FROBIZ": "bilbo"},
1278                 "mounts": {
1279         "/tmp": {"kind": "tmp"},
1280         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1281         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1282         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1283         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1284         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1285     },
1286                 "output_path": "/tmp",
1287                 "priority": 1,
1288                 "runtime_constraints": {}
1289         }`
1290
1291         extraMounts := []string{
1292                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1293                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1294                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1295         }
1296
1297         api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
1298                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1299                 t.logWriter.Close()
1300                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1301         })
1302
1303         c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
1304                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro",
1305                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro",
1306                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro",
1307                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt:/tmp/foo/sub1file2:ro",
1308         })
1309
1310         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1311         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1312         for _, v := range api.Content {
1313                 if v["collection"] != nil {
1314                         c.Check(v["ensure_unique_name"], Equals, true)
1315                         collection := v["collection"].(arvadosclient.Dict)
1316                         if strings.Index(collection["name"].(string), "output") == 0 {
1317                                 manifest := collection["manifest_text"].(string)
1318
1319                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1320 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 9:18:bar 9:18:sub1file2
1321 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1322 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1323 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1324 `)
1325                         }
1326                 }
1327         }
1328 }
1329
1330 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1331         helperRecord := `{
1332                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1333                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1334                 "cwd": "/bin",
1335                 "environment": {"FROBIZ": "bilbo"},
1336                 "mounts": {
1337         "/tmp": {"kind": "tmp"},
1338         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
1339         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1340     },
1341                 "output_path": "/tmp",
1342                 "priority": 1,
1343                 "runtime_constraints": {}
1344         }`
1345
1346         extraMounts := []string{
1347                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1348         }
1349
1350         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
1351                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1352                 t.logWriter.Close()
1353                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1354         })
1355
1356         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1357         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1358         for _, v := range api.Content {
1359                 if v["collection"] != nil {
1360                         collection := v["collection"].(arvadosclient.Dict)
1361                         if strings.Index(collection["name"].(string), "output") == 0 {
1362                                 manifest := collection["manifest_text"].(string)
1363
1364                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1365 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1366 `)
1367                         }
1368                 }
1369         }
1370 }