services/crunch-run/crunchrun_test.go

   1 package main
   2
   3 import (
   4         "bytes"
   5         "crypto/md5"
   6         "encoding/json"
   7         "errors"
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "os"
  12         "os/exec"
  13         "path/filepath"
  14         "runtime/pprof"
  15         "sort"
  16         "strings"
  17         "sync"
  18         "syscall"
  19         "testing"
  20         "time"
  21
  22         "git.curoverse.com/arvados.git/sdk/go/arvados"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         "git.curoverse.com/arvados.git/sdk/go/manifest"
  26         "github.com/curoverse/dockerclient"
  27         . "gopkg.in/check.v1"
  28 )
  29
  30 // Gocheck boilerplate
  31 func TestCrunchExec(t *testing.T) {
  32         TestingT(t)
  33 }
  34
  35 type TestSuite struct{}
  36
  37 // Gocheck boilerplate
  38 var _ = Suite(&TestSuite{})
  39
  40 type ArvTestClient struct {
  41         Total   int64
  42         Calls   int
  43         Content []arvadosclient.Dict
  44         arvados.Container
  45         Logs map[string]*bytes.Buffer
  46         sync.Mutex
  47         WasSetRunning bool
  48 }
  49
  50 type KeepTestClient struct {
  51         Called  bool
  52         Content []byte
  53 }
  54
  55 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
  56 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
  57 var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
  58
  59 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
  60 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
  61
  62 var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
  63 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
  64
  65 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
  66 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
  67
  68 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
  69 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
  70
  71 type TestDockerClient struct {
  72         imageLoaded string
  73         logReader   io.ReadCloser
  74         logWriter   io.WriteCloser
  75         fn          func(t *TestDockerClient)
  76         finish      chan dockerclient.WaitResult
  77         stop        chan bool
  78         cwd         string
  79         env         []string
  80         api         *ArvTestClient
  81 }
  82
  83 func NewTestDockerClient() *TestDockerClient {
  84         t := &TestDockerClient{}
  85         t.logReader, t.logWriter = io.Pipe()
  86         t.finish = make(chan dockerclient.WaitResult)
  87         t.stop = make(chan bool)
  88         t.cwd = "/"
  89         return t
  90 }
  91
  92 func (t *TestDockerClient) StopContainer(id string, timeout int) error {
  93         t.stop <- true
  94         return nil
  95 }
  96
  97 func (t *TestDockerClient) InspectImage(id string) (*dockerclient.ImageInfo, error) {
  98         if t.imageLoaded == id {
  99                 return &dockerclient.ImageInfo{}, nil
 100         } else {
 101                 return nil, errors.New("")
 102         }
 103 }
 104
 105 func (t *TestDockerClient) LoadImage(reader io.Reader) error {
 106         _, err := io.Copy(ioutil.Discard, reader)
 107         if err != nil {
 108                 return err
 109         } else {
 110                 t.imageLoaded = hwImageId
 111                 return nil
 112         }
 113 }
 114
 115 func (t *TestDockerClient) CreateContainer(config *dockerclient.ContainerConfig, name string, authConfig *dockerclient.AuthConfig) (string, error) {
 116         if config.WorkingDir != "" {
 117                 t.cwd = config.WorkingDir
 118         }
 119         t.env = config.Env
 120         return "abcde", nil
 121 }
 122
 123 func (t *TestDockerClient) StartContainer(id string, config *dockerclient.HostConfig) error {
 124         if id == "abcde" {
 125                 go t.fn(t)
 126                 return nil
 127         } else {
 128                 return errors.New("Invalid container id")
 129         }
 130 }
 131
 132 func (t *TestDockerClient) AttachContainer(id string, options *dockerclient.AttachOptions) (io.ReadCloser, error) {
 133         return t.logReader, nil
 134 }
 135
 136 func (t *TestDockerClient) Wait(id string) <-chan dockerclient.WaitResult {
 137         return t.finish
 138 }
 139
 140 func (*TestDockerClient) RemoveImage(name string, force bool) ([]*dockerclient.ImageDelete, error) {
 141         return nil, nil
 142 }
 143
 144 func (client *ArvTestClient) Create(resourceType string,
 145         parameters arvadosclient.Dict,
 146         output interface{}) error {
 147
 148         client.Mutex.Lock()
 149         defer client.Mutex.Unlock()
 150
 151         client.Calls++
 152         client.Content = append(client.Content, parameters)
 153
 154         if resourceType == "logs" {
 155                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 156                 if client.Logs == nil {
 157                         client.Logs = make(map[string]*bytes.Buffer)
 158                 }
 159                 if client.Logs[et] == nil {
 160                         client.Logs[et] = &bytes.Buffer{}
 161                 }
 162                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 163         }
 164
 165         if resourceType == "collections" && output != nil {
 166                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 167                 outmap := output.(*arvados.Collection)
 168                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 169         }
 170
 171         return nil
 172 }
 173
 174 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 175         switch {
 176         case method == "GET" && resourceType == "containers" && action == "auth":
 177                 return json.Unmarshal([]byte(`{
 178                         "kind": "arvados#api_client_authorization",
 179                         "uuid": "`+fakeAuthUUID+`",
 180                         "api_token": "`+fakeAuthToken+`"
 181                         }`), output)
 182         default:
 183                 return fmt.Errorf("Not found")
 184         }
 185 }
 186
 187 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 188         if resourceType == "collections" {
 189                 if uuid == hwPDH {
 190                         output.(*arvados.Collection).ManifestText = hwManifest
 191                 } else if uuid == otherPDH {
 192                         output.(*arvados.Collection).ManifestText = otherManifest
 193                 } else if uuid == normalizedWithSubdirsPDH {
 194                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 195                 } else if uuid == denormalizedWithSubdirsPDH {
 196                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 197                 }
 198         }
 199         if resourceType == "containers" {
 200                 (*output.(*arvados.Container)) = client.Container
 201         }
 202         return nil
 203 }
 204
 205 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 206         client.Mutex.Lock()
 207         defer client.Mutex.Unlock()
 208         client.Calls++
 209         client.Content = append(client.Content, parameters)
 210         if resourceType == "containers" {
 211                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 212                         client.WasSetRunning = true
 213                 }
 214         }
 215         return nil
 216 }
 217
 218 var discoveryMap = map[string]interface{}{"defaultTrashLifetime": float64(1209600)}
 219
 220 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 221         return discoveryMap[key], nil
 222 }
 223
 224 // CalledWith returns the parameters from the first API call whose
 225 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 226 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 227 // no call matches, it returns nil.
 228 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 229 call:
 230         for _, content := range client.Content {
 231                 var v interface{} = content
 232                 for _, k := range strings.Split(jpath, ".") {
 233                         if dict, ok := v.(arvadosclient.Dict); !ok {
 234                                 continue call
 235                         } else {
 236                                 v = dict[k]
 237                         }
 238                 }
 239                 if v == expect {
 240                         return content
 241                 }
 242         }
 243         return nil
 244 }
 245
 246 func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 247         client.Content = buf
 248         return fmt.Sprintf("%s+%d", hash, len(buf)), len(buf), nil
 249 }
 250
 251 type FileWrapper struct {
 252         io.ReadCloser
 253         len uint64
 254 }
 255
 256 func (fw FileWrapper) Len() uint64 {
 257         return fw.len
 258 }
 259
 260 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 261         return 0, errors.New("not implemented")
 262 }
 263
 264 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 265         if filename == hwImageId+".tar" {
 266                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 267                 client.Called = true
 268                 return FileWrapper{rdr, 1321984}, nil
 269         }
 270         return nil, nil
 271 }
 272
 273 func (s *TestSuite) TestLoadImage(c *C) {
 274         kc := &KeepTestClient{}
 275         docker := NewTestDockerClient()
 276         cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 277
 278         _, err := cr.Docker.RemoveImage(hwImageId, true)
 279
 280         _, err = cr.Docker.InspectImage(hwImageId)
 281         c.Check(err, NotNil)
 282
 283         cr.Container.ContainerImage = hwPDH
 284
 285         // (1) Test loading image from keep
 286         c.Check(kc.Called, Equals, false)
 287         c.Check(cr.ContainerConfig.Image, Equals, "")
 288
 289         err = cr.LoadImage()
 290
 291         c.Check(err, IsNil)
 292         defer func() {
 293                 cr.Docker.RemoveImage(hwImageId, true)
 294         }()
 295
 296         c.Check(kc.Called, Equals, true)
 297         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 298
 299         _, err = cr.Docker.InspectImage(hwImageId)
 300         c.Check(err, IsNil)
 301
 302         // (2) Test using image that's already loaded
 303         kc.Called = false
 304         cr.ContainerConfig.Image = ""
 305
 306         err = cr.LoadImage()
 307         c.Check(err, IsNil)
 308         c.Check(kc.Called, Equals, false)
 309         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 310
 311 }
 312
 313 type ArvErrorTestClient struct{}
 314
 315 func (ArvErrorTestClient) Create(resourceType string,
 316         parameters arvadosclient.Dict,
 317         output interface{}) error {
 318         return nil
 319 }
 320
 321 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 322         return errors.New("ArvError")
 323 }
 324
 325 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 326         return errors.New("ArvError")
 327 }
 328
 329 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 330         return nil
 331 }
 332
 333 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 334         return discoveryMap[key], nil
 335 }
 336
 337 type KeepErrorTestClient struct{}
 338
 339 func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 340         return "", 0, errors.New("KeepError")
 341 }
 342
 343 func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 344         return nil, errors.New("KeepError")
 345 }
 346
 347 type KeepReadErrorTestClient struct{}
 348
 349 func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 350         return "", 0, nil
 351 }
 352
 353 type ErrorReader struct{}
 354
 355 func (ErrorReader) Read(p []byte) (n int, err error) {
 356         return 0, errors.New("ErrorReader")
 357 }
 358
 359 func (ErrorReader) Close() error {
 360         return nil
 361 }
 362
 363 func (ErrorReader) Len() uint64 {
 364         return 0
 365 }
 366
 367 func (ErrorReader) Seek(int64, int) (int64, error) {
 368         return 0, errors.New("ErrorReader")
 369 }
 370
 371 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 372         return ErrorReader{}, nil
 373 }
 374
 375 func (s *TestSuite) TestLoadImageArvError(c *C) {
 376         // (1) Arvados error
 377         cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 378         cr.Container.ContainerImage = hwPDH
 379
 380         err := cr.LoadImage()
 381         c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
 382 }
 383
 384 func (s *TestSuite) TestLoadImageKeepError(c *C) {
 385         // (2) Keep error
 386         docker := NewTestDockerClient()
 387         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 388         cr.Container.ContainerImage = hwPDH
 389
 390         err := cr.LoadImage()
 391         c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError")
 392 }
 393
 394 func (s *TestSuite) TestLoadImageCollectionError(c *C) {
 395         // (3) Collection doesn't contain image
 396         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 397         cr.Container.ContainerImage = otherPDH
 398
 399         err := cr.LoadImage()
 400         c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
 401 }
 402
 403 func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
 404         // (4) Collection doesn't contain image
 405         docker := NewTestDockerClient()
 406         cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 407         cr.Container.ContainerImage = hwPDH
 408
 409         err := cr.LoadImage()
 410         c.Check(err, NotNil)
 411 }
 412
 413 type ClosableBuffer struct {
 414         bytes.Buffer
 415 }
 416
 417 func (*ClosableBuffer) Close() error {
 418         return nil
 419 }
 420
 421 type TestLogs struct {
 422         Stdout ClosableBuffer
 423         Stderr ClosableBuffer
 424 }
 425
 426 func (tl *TestLogs) NewTestLoggingWriter(logstr string) io.WriteCloser {
 427         if logstr == "stdout" {
 428                 return &tl.Stdout
 429         }
 430         if logstr == "stderr" {
 431                 return &tl.Stderr
 432         }
 433         return nil
 434 }
 435
 436 func dockerLog(fd byte, msg string) []byte {
 437         by := []byte(msg)
 438         header := make([]byte, 8+len(by))
 439         header[0] = fd
 440         header[7] = byte(len(by))
 441         copy(header[8:], by)
 442         return header
 443 }
 444
 445 func (s *TestSuite) TestRunContainer(c *C) {
 446         docker := NewTestDockerClient()
 447         docker.fn = func(t *TestDockerClient) {
 448                 t.logWriter.Write(dockerLog(1, "Hello world\n"))
 449                 t.logWriter.Close()
 450                 t.finish <- dockerclient.WaitResult{}
 451         }
 452         cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 453
 454         var logs TestLogs
 455         cr.NewLogWriter = logs.NewTestLoggingWriter
 456         cr.Container.ContainerImage = hwPDH
 457         cr.Container.Command = []string{"./hw"}
 458         err := cr.LoadImage()
 459         c.Check(err, IsNil)
 460
 461         err = cr.CreateContainer()
 462         c.Check(err, IsNil)
 463
 464         err = cr.StartContainer()
 465         c.Check(err, IsNil)
 466
 467         err = cr.WaitFinish()
 468         c.Check(err, IsNil)
 469
 470         c.Check(strings.HasSuffix(logs.Stdout.String(), "Hello world\n"), Equals, true)
 471         c.Check(logs.Stderr.String(), Equals, "")
 472 }
 473
 474 func (s *TestSuite) TestCommitLogs(c *C) {
 475         api := &ArvTestClient{}
 476         kc := &KeepTestClient{}
 477         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 478         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 479
 480         cr.CrunchLog.Print("Hello world!")
 481         cr.CrunchLog.Print("Goodbye")
 482         cr.finalState = "Complete"
 483
 484         err := cr.CommitLogs()
 485         c.Check(err, IsNil)
 486
 487         c.Check(api.Calls, Equals, 2)
 488         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 489         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 490         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 491         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 492 }
 493
 494 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 495         api := &ArvTestClient{}
 496         kc := &KeepTestClient{}
 497         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 498
 499         err := cr.UpdateContainerRunning()
 500         c.Check(err, IsNil)
 501
 502         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 503 }
 504
 505 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 506         api := &ArvTestClient{}
 507         kc := &KeepTestClient{}
 508         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 509
 510         cr.LogsPDH = new(string)
 511         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 512
 513         cr.ExitCode = new(int)
 514         *cr.ExitCode = 42
 515         cr.finalState = "Complete"
 516
 517         err := cr.UpdateContainerFinal()
 518         c.Check(err, IsNil)
 519
 520         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 521         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 522         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 523 }
 524
 525 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 526         api := &ArvTestClient{}
 527         kc := &KeepTestClient{}
 528         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 529         cr.cCancelled = true
 530         cr.finalState = "Cancelled"
 531
 532         err := cr.UpdateContainerFinal()
 533         c.Check(err, IsNil)
 534
 535         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 536         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 537         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 538 }
 539
 540 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 541 // dress rehearsal of the Run() function, starting from a JSON container record.
 542 func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
 543         rec := arvados.Container{}
 544         err := json.Unmarshal([]byte(record), &rec)
 545         c.Check(err, IsNil)
 546
 547         docker := NewTestDockerClient()
 548         docker.fn = fn
 549         docker.RemoveImage(hwImageId, true)
 550
 551         api = &ArvTestClient{Container: rec}
 552         docker.api = api
 553         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 554         cr.statInterval = 100 * time.Millisecond
 555         am := &ArvMountCmdLine{}
 556         cr.RunArvMount = am.ArvMountTest
 557
 558         realTemp, err = ioutil.TempDir("", "crunchrun_test1-")
 559         c.Assert(err, IsNil)
 560         defer os.RemoveAll(realTemp)
 561
 562         tempcount := 0
 563         cr.MkTempDir = func(_ string, prefix string) (string, error) {
 564                 tempcount++
 565                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 566                 err := os.Mkdir(d, os.ModePerm)
 567                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 568                         // Test case must have pre-populated the tempdir
 569                         err = nil
 570                 }
 571                 return d, err
 572         }
 573
 574         if extraMounts != nil && len(extraMounts) > 0 {
 575                 err := cr.SetupArvMountPoint("keep")
 576                 c.Check(err, IsNil)
 577
 578                 for _, m := range extraMounts {
 579                         os.MkdirAll(cr.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 580                 }
 581         }
 582
 583         err = cr.Run()
 584         c.Check(err, IsNil)
 585         c.Check(api.WasSetRunning, Equals, true)
 586
 587         c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil)
 588
 589         if err != nil {
 590                 for k, v := range api.Logs {
 591                         c.Log(k)
 592                         c.Log(v.String())
 593                 }
 594         }
 595
 596         return
 597 }
 598
 599 func (s *TestSuite) TestFullRunHello(c *C) {
 600         api, _, _ := FullRunHelper(c, `{
 601     "command": ["echo", "hello world"],
 602     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 603     "cwd": ".",
 604     "environment": {},
 605     "mounts": {"/tmp": {"kind": "tmp"} },
 606     "output_path": "/tmp",
 607     "priority": 1,
 608     "runtime_constraints": {}
 609 }`, nil, func(t *TestDockerClient) {
 610                 t.logWriter.Write(dockerLog(1, "hello world\n"))
 611                 t.logWriter.Close()
 612                 t.finish <- dockerclient.WaitResult{}
 613         })
 614
 615         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 616         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 617         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello world\n"), Equals, true)
 618
 619 }
 620
 621 func (s *TestSuite) TestCrunchstat(c *C) {
 622         api, _, _ := FullRunHelper(c, `{
 623                 "command": ["sleep", "1"],
 624                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 625                 "cwd": ".",
 626                 "environment": {},
 627                 "mounts": {"/tmp": {"kind": "tmp"} },
 628                 "output_path": "/tmp",
 629                 "priority": 1,
 630                 "runtime_constraints": {}
 631         }`, nil, func(t *TestDockerClient) {
 632                 time.Sleep(time.Second)
 633                 t.logWriter.Close()
 634                 t.finish <- dockerclient.WaitResult{}
 635         })
 636
 637         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 638         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 639
 640         // We didn't actually start a container, so crunchstat didn't
 641         // find accounting files and therefore didn't log any stats.
 642         // It should have logged a "can't find accounting files"
 643         // message after one poll interval, though, so we can confirm
 644         // it's alive:
 645         c.Assert(api.Logs["crunchstat"], NotNil)
 646         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 647
 648         // The "files never appeared" log assures us that we called
 649         // (*crunchstat.Reporter)Stop(), and that we set it up with
 650         // the correct container ID "abcde":
 651         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
 652 }
 653
 654 func (s *TestSuite) TestNodeInfo(c *C) {
 655         api, _, _ := FullRunHelper(c, `{
 656                 "command": ["sleep", "1"],
 657                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 658                 "cwd": ".",
 659                 "environment": {},
 660                 "mounts": {"/tmp": {"kind": "tmp"} },
 661                 "output_path": "/tmp",
 662                 "priority": 1,
 663                 "runtime_constraints": {}
 664         }`, nil, func(t *TestDockerClient) {
 665                 time.Sleep(time.Second)
 666                 t.logWriter.Close()
 667                 t.finish <- dockerclient.WaitResult{}
 668         })
 669
 670         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 671         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 672
 673         c.Assert(api.Logs["node-info"], NotNil)
 674         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
 675         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
 676         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
 677         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
 678 }
 679
 680 func (s *TestSuite) TestFullRunStderr(c *C) {
 681         api, _, _ := FullRunHelper(c, `{
 682     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 683     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 684     "cwd": ".",
 685     "environment": {},
 686     "mounts": {"/tmp": {"kind": "tmp"} },
 687     "output_path": "/tmp",
 688     "priority": 1,
 689     "runtime_constraints": {}
 690 }`, nil, func(t *TestDockerClient) {
 691                 t.logWriter.Write(dockerLog(1, "hello\n"))
 692                 t.logWriter.Write(dockerLog(2, "world\n"))
 693                 t.logWriter.Close()
 694                 t.finish <- dockerclient.WaitResult{ExitCode: 1}
 695         })
 696
 697         final := api.CalledWith("container.state", "Complete")
 698         c.Assert(final, NotNil)
 699         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 700         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 701
 702         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello\n"), Equals, true)
 703         c.Check(strings.HasSuffix(api.Logs["stderr"].String(), "world\n"), Equals, true)
 704 }
 705
 706 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 707         api, _, _ := FullRunHelper(c, `{
 708     "command": ["pwd"],
 709     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 710     "cwd": ".",
 711     "environment": {},
 712     "mounts": {"/tmp": {"kind": "tmp"} },
 713     "output_path": "/tmp",
 714     "priority": 1,
 715     "runtime_constraints": {}
 716 }`, nil, func(t *TestDockerClient) {
 717                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 718                 t.logWriter.Close()
 719                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 720         })
 721
 722         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 723         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 724         c.Log(api.Logs["stdout"])
 725         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/\n"), Equals, true)
 726 }
 727
 728 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 729         api, _, _ := FullRunHelper(c, `{
 730     "command": ["pwd"],
 731     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 732     "cwd": "/bin",
 733     "environment": {},
 734     "mounts": {"/tmp": {"kind": "tmp"} },
 735     "output_path": "/tmp",
 736     "priority": 1,
 737     "runtime_constraints": {}
 738 }`, nil, func(t *TestDockerClient) {
 739                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 740                 t.logWriter.Close()
 741                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 742         })
 743
 744         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 745         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 746         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/bin\n"), Equals, true)
 747 }
 748
 749 func (s *TestSuite) TestStopOnSignal(c *C) {
 750         s.testStopContainer(c, func(cr *ContainerRunner) {
 751                 go func() {
 752                         for !cr.cStarted {
 753                                 time.Sleep(time.Millisecond)
 754                         }
 755                         cr.SigChan <- syscall.SIGINT
 756                 }()
 757         })
 758 }
 759
 760 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 761         s.testStopContainer(c, func(cr *ContainerRunner) {
 762                 cr.ArvMountExit = make(chan error)
 763                 go func() {
 764                         cr.ArvMountExit <- exec.Command("true").Run()
 765                         close(cr.ArvMountExit)
 766                 }()
 767         })
 768 }
 769
 770 func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
 771         record := `{
 772     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 773     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 774     "cwd": ".",
 775     "environment": {},
 776     "mounts": {"/tmp": {"kind": "tmp"} },
 777     "output_path": "/tmp",
 778     "priority": 1,
 779     "runtime_constraints": {}
 780 }`
 781
 782         rec := arvados.Container{}
 783         err := json.Unmarshal([]byte(record), &rec)
 784         c.Check(err, IsNil)
 785
 786         docker := NewTestDockerClient()
 787         docker.fn = func(t *TestDockerClient) {
 788                 <-t.stop
 789                 t.logWriter.Write(dockerLog(1, "foo\n"))
 790                 t.logWriter.Close()
 791                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 792         }
 793         docker.RemoveImage(hwImageId, true)
 794
 795         api := &ArvTestClient{Container: rec}
 796         cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 797         cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 798         setup(cr)
 799
 800         done := make(chan error)
 801         go func() {
 802                 done <- cr.Run()
 803         }()
 804         select {
 805         case <-time.After(20 * time.Second):
 806                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
 807                 c.Fatal("timed out")
 808         case err = <-done:
 809                 c.Check(err, IsNil)
 810         }
 811         for k, v := range api.Logs {
 812                 c.Log(k)
 813                 c.Log(v.String())
 814         }
 815
 816         c.Check(api.CalledWith("container.log", nil), NotNil)
 817         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
 818         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "foo\n"), Equals, true)
 819 }
 820
 821 func (s *TestSuite) TestFullRunSetEnv(c *C) {
 822         api, _, _ := FullRunHelper(c, `{
 823     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
 824     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 825     "cwd": "/bin",
 826     "environment": {"FROBIZ": "bilbo"},
 827     "mounts": {"/tmp": {"kind": "tmp"} },
 828     "output_path": "/tmp",
 829     "priority": 1,
 830     "runtime_constraints": {}
 831 }`, nil, func(t *TestDockerClient) {
 832                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 833                 t.logWriter.Close()
 834                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
 835         })
 836
 837         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 838         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 839         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "bilbo\n"), Equals, true)
 840 }
 841
 842 type ArvMountCmdLine struct {
 843         Cmd   []string
 844         token string
 845 }
 846
 847 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
 848         am.Cmd = c
 849         am.token = token
 850         return nil, nil
 851 }
 852
 853 func stubCert(temp string) string {
 854         path := temp + "/ca-certificates.crt"
 855         crt, _ := os.Create(path)
 856         crt.Close()
 857         arvadosclient.CertFiles = []string{path}
 858         return path
 859 }
 860
 861 func (s *TestSuite) TestSetupMounts(c *C) {
 862         api := &ArvTestClient{}
 863         kc := &KeepTestClient{}
 864         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 865         am := &ArvMountCmdLine{}
 866         cr.RunArvMount = am.ArvMountTest
 867
 868         realTemp, err := ioutil.TempDir("", "crunchrun_test1-")
 869         c.Assert(err, IsNil)
 870         certTemp, err := ioutil.TempDir("", "crunchrun_test2-")
 871         c.Assert(err, IsNil)
 872         stubCertPath := stubCert(certTemp)
 873
 874         defer os.RemoveAll(realTemp)
 875         defer os.RemoveAll(certTemp)
 876
 877         i := 0
 878         cr.MkTempDir = func(_ string, prefix string) (string, error) {
 879                 i++
 880                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
 881                 err := os.Mkdir(d, os.ModePerm)
 882                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 883                         // Test case must have pre-populated the tempdir
 884                         err = nil
 885                 }
 886                 return d, err
 887         }
 888
 889         checkEmpty := func() {
 890                 filepath.Walk(realTemp, func(path string, _ os.FileInfo, err error) error {
 891                         c.Check(path, Equals, realTemp)
 892                         c.Check(err, IsNil)
 893                         return nil
 894                 })
 895         }
 896
 897         {
 898                 i = 0
 899                 cr.ArvMountPoint = ""
 900                 cr.Container.Mounts = make(map[string]arvados.Mount)
 901                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
 902                 cr.OutputPath = "/tmp"
 903
 904                 err := cr.SetupMounts()
 905                 c.Check(err, IsNil)
 906                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 907                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp"})
 908                 cr.CleanupDirs()
 909                 checkEmpty()
 910         }
 911
 912         {
 913                 i = 0
 914                 cr.ArvMountPoint = ""
 915                 cr.Container.Mounts = make(map[string]arvados.Mount)
 916                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
 917                 cr.OutputPath = "/tmp"
 918
 919                 apiflag := true
 920                 cr.Container.RuntimeConstraints.API = &apiflag
 921
 922                 err := cr.SetupMounts()
 923                 c.Check(err, IsNil)
 924                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 925                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"})
 926                 cr.CleanupDirs()
 927                 checkEmpty()
 928
 929                 apiflag = false
 930         }
 931
 932         {
 933                 i = 0
 934                 cr.ArvMountPoint = ""
 935                 cr.Container.Mounts = map[string]arvados.Mount{
 936                         "/keeptmp": {Kind: "collection", Writable: true},
 937                 }
 938                 cr.OutputPath = "/keeptmp"
 939
 940                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
 941
 942                 err := cr.SetupMounts()
 943                 c.Check(err, IsNil)
 944                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 945                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/tmp0:/keeptmp"})
 946                 cr.CleanupDirs()
 947                 checkEmpty()
 948         }
 949
 950         {
 951                 i = 0
 952                 cr.ArvMountPoint = ""
 953                 cr.Container.Mounts = map[string]arvados.Mount{
 954                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
 955                         "/keepout": {Kind: "collection", Writable: true},
 956                 }
 957                 cr.OutputPath = "/keepout"
 958
 959                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
 960                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
 961
 962                 err := cr.SetupMounts()
 963                 c.Check(err, IsNil)
 964                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 965                 sort.StringSlice(cr.Binds).Sort()
 966                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
 967                         realTemp + "/keep1/tmp0:/keepout"})
 968                 cr.CleanupDirs()
 969                 checkEmpty()
 970         }
 971
 972         {
 973                 i = 0
 974                 cr.ArvMountPoint = ""
 975                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
 976                 cr.Container.Mounts = map[string]arvados.Mount{
 977                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
 978                         "/keepout": {Kind: "collection", Writable: true},
 979                 }
 980                 cr.OutputPath = "/keepout"
 981
 982                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
 983                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
 984
 985                 err := cr.SetupMounts()
 986                 c.Check(err, IsNil)
 987                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 988                 sort.StringSlice(cr.Binds).Sort()
 989                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
 990                         realTemp + "/keep1/tmp0:/keepout"})
 991                 cr.CleanupDirs()
 992                 checkEmpty()
 993         }
 994
 995         for _, test := range []struct {
 996                 in  interface{}
 997                 out string
 998         }{
 999                 {in: "foo", out: `"foo"`},
1000                 {in: nil, out: `null`},
1001                 {in: map[string]int{"foo": 123}, out: `{"foo":123}`},
1002         } {
1003                 i = 0
1004                 cr.ArvMountPoint = ""
1005                 cr.Container.Mounts = map[string]arvados.Mount{
1006                         "/mnt/test.json": {Kind: "json", Content: test.in},
1007                 }
1008                 err := cr.SetupMounts()
1009                 c.Check(err, IsNil)
1010                 sort.StringSlice(cr.Binds).Sort()
1011                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2/mountdata.json:/mnt/test.json:ro"})
1012                 content, err := ioutil.ReadFile(realTemp + "/2/mountdata.json")
1013                 c.Check(err, IsNil)
1014                 c.Check(content, DeepEquals, []byte(test.out))
1015                 cr.CleanupDirs()
1016                 checkEmpty()
1017         }
1018
1019         // Read-only mount points are allowed underneath output_dir mount point
1020         {
1021                 i = 0
1022                 cr.ArvMountPoint = ""
1023                 cr.Container.Mounts = make(map[string]arvados.Mount)
1024                 cr.Container.Mounts = map[string]arvados.Mount{
1025                         "/tmp":     {Kind: "tmp"},
1026                         "/tmp/foo": {Kind: "collection"},
1027                 }
1028                 cr.OutputPath = "/tmp"
1029
1030                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1031
1032                 err := cr.SetupMounts()
1033                 c.Check(err, IsNil)
1034                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1035                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"})
1036                 cr.CleanupDirs()
1037                 checkEmpty()
1038         }
1039
1040         // Writable mount points are not allowed underneath output_dir mount point
1041         {
1042                 i = 0
1043                 cr.ArvMountPoint = ""
1044                 cr.Container.Mounts = make(map[string]arvados.Mount)
1045                 cr.Container.Mounts = map[string]arvados.Mount{
1046                         "/tmp":     {Kind: "tmp"},
1047                         "/tmp/foo": {Kind: "collection", Writable: true},
1048                 }
1049                 cr.OutputPath = "/tmp"
1050
1051                 err := cr.SetupMounts()
1052                 c.Check(err, NotNil)
1053                 c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
1054                 cr.CleanupDirs()
1055                 checkEmpty()
1056         }
1057
1058         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1059         {
1060                 i = 0
1061                 cr.ArvMountPoint = ""
1062                 cr.Container.Mounts = make(map[string]arvados.Mount)
1063                 cr.Container.Mounts = map[string]arvados.Mount{
1064                         "/tmp":     {Kind: "tmp"},
1065                         "/tmp/foo": {Kind: "json"},
1066                 }
1067                 cr.OutputPath = "/tmp"
1068
1069                 err := cr.SetupMounts()
1070                 c.Check(err, NotNil)
1071                 c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`)
1072                 cr.CleanupDirs()
1073                 checkEmpty()
1074         }
1075 }
1076
1077 func (s *TestSuite) TestStdout(c *C) {
1078         helperRecord := `{
1079                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1080                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1081                 "cwd": "/bin",
1082                 "environment": {"FROBIZ": "bilbo"},
1083                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1084                 "output_path": "/tmp",
1085                 "priority": 1,
1086                 "runtime_constraints": {}
1087         }`
1088
1089         api, _, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
1090                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1091                 t.logWriter.Close()
1092                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1093         })
1094
1095         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1096         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1097         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1098 }
1099
1100 // Used by the TestStdoutWithWrongPath*()
1101 func StdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, err error) {
1102         rec := arvados.Container{}
1103         err = json.Unmarshal([]byte(record), &rec)
1104         c.Check(err, IsNil)
1105
1106         docker := NewTestDockerClient()
1107         docker.fn = fn
1108         docker.RemoveImage(hwImageId, true)
1109
1110         api = &ArvTestClient{Container: rec}
1111         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
1112         am := &ArvMountCmdLine{}
1113         cr.RunArvMount = am.ArvMountTest
1114
1115         err = cr.Run()
1116         return
1117 }
1118
1119 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1120         _, _, err := StdoutErrorRunHelper(c, `{
1121     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1122     "output_path": "/tmp"
1123 }`, func(t *TestDockerClient) {})
1124
1125         c.Check(err, NotNil)
1126         c.Check(strings.Contains(err.Error(), "Stdout path does not start with OutputPath"), Equals, true)
1127 }
1128
1129 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1130         _, _, err := StdoutErrorRunHelper(c, `{
1131     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1132     "output_path": "/tmp"
1133 }`, func(t *TestDockerClient) {})
1134
1135         c.Check(err, NotNil)
1136         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
1137 }
1138
1139 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1140         _, _, err := StdoutErrorRunHelper(c, `{
1141     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1142     "output_path": "/tmp"
1143 }`, func(t *TestDockerClient) {})
1144
1145         c.Check(err, NotNil)
1146         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
1147 }
1148
1149 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1150         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1151         defer os.Unsetenv("ARVADOS_API_HOST")
1152         api, _, _ := FullRunHelper(c, `{
1153     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1154     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1155     "cwd": "/bin",
1156     "environment": {},
1157     "mounts": {"/tmp": {"kind": "tmp"} },
1158     "output_path": "/tmp",
1159     "priority": 1,
1160     "runtime_constraints": {"API": true}
1161 }`, nil, func(t *TestDockerClient) {
1162                 t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
1163                 t.logWriter.Close()
1164                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1165         })
1166
1167         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1168         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1169         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "test.arvados.org\n"), Equals, true)
1170         c.Check(api.CalledWith("container.output", "d41d8cd98f00b204e9800998ecf8427e+0"), NotNil)
1171 }
1172
1173 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1174         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1175         defer os.Unsetenv("ARVADOS_API_HOST")
1176         api, _, _ := FullRunHelper(c, `{
1177     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1178     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1179     "cwd": "/bin",
1180     "environment": {},
1181     "mounts": {"/tmp": {"kind": "tmp"} },
1182     "output_path": "/tmp",
1183     "priority": 1,
1184     "runtime_constraints": {"API": true}
1185 }`, nil, func(t *TestDockerClient) {
1186                 t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
1187                 t.logWriter.Close()
1188                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1189         })
1190
1191         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1192         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1193         c.Check(api.CalledWith("container.output", "d4ab34d3d4f8a72f5c4973051ae69fab+122"), NotNil)
1194 }
1195
1196 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1197         helperRecord := `{
1198                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1199                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1200                 "cwd": "/bin",
1201                 "environment": {"FROBIZ": "bilbo"},
1202                 "mounts": {
1203         "/tmp": {"kind": "tmp"},
1204         "/tmp/foo": {"kind": "collection",
1205                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1206                      "exclude_from_output": true
1207         },
1208         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1209     },
1210                 "output_path": "/tmp",
1211                 "priority": 1,
1212                 "runtime_constraints": {}
1213         }`
1214
1215         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1216
1217         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
1218                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1219                 t.logWriter.Close()
1220                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1221         })
1222
1223         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1224         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1225         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1226 }
1227
1228 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1229         helperRecord := `{
1230                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1231                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1232                 "cwd": "/bin",
1233                 "environment": {"FROBIZ": "bilbo"},
1234                 "mounts": {
1235         "/tmp": {"kind": "tmp"},
1236         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1237         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1238         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1239         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1240         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1241     },
1242                 "output_path": "/tmp",
1243                 "priority": 1,
1244                 "runtime_constraints": {}
1245         }`
1246
1247         extraMounts := []string{
1248                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1249                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1250                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1251         }
1252
1253         api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
1254                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1255                 t.logWriter.Close()
1256                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1257         })
1258
1259         c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
1260                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro",
1261                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro",
1262                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro",
1263                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt:/tmp/foo/sub1file2:ro",
1264         })
1265
1266         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1267         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1268         for _, v := range api.Content {
1269                 if v["collection"] != nil {
1270                         c.Check(v["ensure_unique_name"], Equals, true)
1271                         collection := v["collection"].(arvadosclient.Dict)
1272                         if strings.Index(collection["name"].(string), "output") == 0 {
1273                                 manifest := collection["manifest_text"].(string)
1274
1275                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1276 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 9:18:bar 9:18:sub1file2
1277 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1278 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1279 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1280 `)
1281                         }
1282                 }
1283         }
1284 }
1285
1286 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1287         helperRecord := `{
1288                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1289                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1290                 "cwd": "/bin",
1291                 "environment": {"FROBIZ": "bilbo"},
1292                 "mounts": {
1293         "/tmp": {"kind": "tmp"},
1294         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
1295         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1296     },
1297                 "output_path": "/tmp",
1298                 "priority": 1,
1299                 "runtime_constraints": {}
1300         }`
1301
1302         extraMounts := []string{
1303                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1304         }
1305
1306         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
1307                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1308                 t.logWriter.Close()
1309                 t.finish <- dockerclient.WaitResult{ExitCode: 0}
1310         })
1311
1312         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1313         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1314         for _, v := range api.Content {
1315                 if v["collection"] != nil {
1316                         collection := v["collection"].(arvadosclient.Dict)
1317                         if strings.Index(collection["name"].(string), "output") == 0 {
1318                                 manifest := collection["manifest_text"].(string)
1319
1320                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1321 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1322 `)
1323                         }
1324                 }
1325         }
1326 }