14406: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --combined services/crunch-run/crunchrun.go
index 800556866a43e985c9107370ff83d9691158aef1,d055106d3530cadcc4ffce4ab14b490d28cddcfd..1deb74031667d7ade04968344d3b262b3ccf1dd1
--- 1/services/crunch-run/crunchrun.go
--- 2/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@@ -61,7 -61,6 +61,7 @@@ type IKeepClient interface 
        PutB(buf []byte) (string, int, error)
        ReadAt(locator string, p []byte, off int) (int, error)
        ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error)
 +      LocalLocator(locator string) (string, error)
        ClearBlockCache()
  }
  
@@@ -123,7 -122,7 +123,7 @@@ type ContainerRunner struct 
        SigChan         chan os.Signal
        ArvMountExit    chan error
        SecretMounts    map[string]arvados.Mount
-       MkArvClient     func(token string) (IArvadosClient, error)
+       MkArvClient     func(token string) (IArvadosClient, IKeepClient, error)
        finalState      string
        parentTemp      string
  
@@@ -238,8 -237,17 +238,17 @@@ func (runner *ContainerRunner) LoadImag
  
        runner.CrunchLog.Printf("Fetching Docker image from collection '%s'", runner.Container.ContainerImage)
  
+       tok, err := runner.ContainerToken()
+       if err != nil {
+               return fmt.Errorf("While getting container token (LoadImage): %v", err)
+       }
+       arvClient, kc, err := runner.MkArvClient(tok)
+       if err != nil {
+               return fmt.Errorf("While creating arv client (LoadImage): %v", err)
+       }
+ 
        var collection arvados.Collection
-       err = runner.ArvClient.Get("collections", runner.Container.ContainerImage, nil, &collection)
+       err = arvClient.Get("collections", runner.Container.ContainerImage, nil, &collection)
        if err != nil {
                return fmt.Errorf("While getting container image collection: %v", err)
        }
@@@ -260,7 -268,7 +269,7 @@@
                runner.CrunchLog.Print("Loading Docker image from keep")
  
                var readCloser io.ReadCloser
-               readCloser, err = runner.Kc.ManifestFileReader(manifest, img)
+               readCloser, err = kc.ManifestFileReader(manifest, img)
                if err != nil {
                        return fmt.Errorf("While creating ManifestFileReader for container image: %v", err)
                }
@@@ -282,7 -290,7 +291,7 @@@
  
        runner.ContainerConfig.Image = imageID
  
-       runner.Kc.ClearBlockCache()
+       kc.ClearBlockCache()
  
        return nil
  }
@@@ -1294,17 -1302,6 +1303,17 @@@ func (runner *ContainerRunner) CaptureO
        if err != nil {
                return err
        }
 +      if n := len(regexp.MustCompile(` [0-9a-f]+\+\S*\+R`).FindAllStringIndex(txt, -1)); n > 0 {
 +              runner.CrunchLog.Printf("Copying %d data blocks from remote input collections...", n)
 +              fs, err := (&arvados.Collection{ManifestText: txt}).FileSystem(runner.client, runner.Kc)
 +              if err != nil {
 +                      return err
 +              }
 +              txt, err = fs.MarshalManifest(".")
 +              if err != nil {
 +                      return err
 +              }
 +      }
        var resp arvados.Collection
        err = runner.ArvClient.Create("collections", arvadosclient.Dict{
                "ensure_unique_name": true,
@@@ -1691,7 -1688,7 +1700,7 @@@ func (runner *ContainerRunner) fetchCon
                return fmt.Errorf("error getting container token: %v", err)
        }
  
-       containerClient, err := runner.MkArvClient(containerToken)
+       containerClient, _, err := runner.MkArvClient(containerToken)
        if err != nil {
                return fmt.Errorf("error creating container API client: %v", err)
        }
@@@ -1731,13 -1728,17 +1740,17 @@@ func NewContainerRunner(client *arvados
                }
                return ps, nil
        }
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
                cl, err := arvadosclient.MakeArvadosClient()
                if err != nil {
-                       return nil, err
+                       return nil, nil, err
                }
                cl.ApiToken = token
-               return cl, nil
+               kc, err := keepclient.MakeKeepClient(cl)
+               if err != nil {
+                       return nil, nil, err
+               }
+               return cl, kc, nil
        }
        var err error
        cr.LogCollection, err = (&arvados.Collection{}).FileSystem(cr.client, cr.Kc)

diff --combined services/crunch-run/crunchrun_test.go
index ab6afc77cbcf82855e4cd11920589d94d9968504,2f254b5bd71fa556869392b58f0b8a102f22e3ce..0df048cc8b95000fbb214dd88cabd83c6b9f71d1
--- 1/services/crunch-run/crunchrun_test.go
--- 2/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@@ -375,10 -375,6 +375,10 @@@ call
        return nil
  }
  
 +func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
 +      return locator, nil
 +}
 +
  func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
        client.Content = buf
        return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
@@@ -447,6 -443,10 +447,10 @@@ func (s *TestSuite) TestLoadImage(c *C
        cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
  
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{}, kc, nil
+       }
+ 
        _, err = cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
        c.Check(err, IsNil)
  
@@@ -492,6 -492,9 +496,9 @@@ func (ArvErrorTestClient) Create(resour
  }
  
  func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
+       if method == "GET" && resourceType == "containers" && action == "auth" {
+               return nil
+       }
        return errors.New("ArvError")
  }
  
@@@ -524,10 -527,6 +531,10 @@@ func (*KeepErrorTestClient) PutB(buf []
        return "", 0, errors.New("KeepError")
  }
  
 +func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
 +      return "", errors.New("KeepError")
 +}
 +
  type KeepReadErrorTestClient struct {
        KeepTestClient
  }
@@@ -556,9 -555,13 +563,13 @@@ func (s *TestSuite) TestLoadImageArvErr
        // (1) Arvados error
        kc := &KeepTestClient{}
        defer kc.Close()
-       cr, err := NewContainerRunner(s.client, ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       cr, err := NewContainerRunner(s.client, &ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
+ 
        cr.Container.ContainerImage = hwPDH
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvErrorTestClient{}, &KeepTestClient{}, nil
+       }
  
        err = cr.LoadImage()
        c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
@@@ -566,9 -569,13 +577,13 @@@
  
  func (s *TestSuite) TestLoadImageKeepError(c *C) {
        // (2) Keep error
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       kc := &KeepErrorTestClient{}
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.Container.ContainerImage = hwPDH
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{}, kc, nil
+       }
  
        err = cr.LoadImage()
        c.Assert(err, NotNil)
@@@ -577,9 -584,13 +592,13 @@@
  
  func (s *TestSuite) TestLoadImageCollectionError(c *C) {
        // (3) Collection doesn't contain image
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepReadErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       kc := &KeepReadErrorTestClient{}
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.Container.ContainerImage = otherPDH
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{}, kc, nil
+       }
  
        err = cr.LoadImage()
        c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
@@@ -587,9 -598,13 +606,13 @@@
  
  func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
        // (4) Collection doesn't contain image
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepReadErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       kc := &KeepReadErrorTestClient{}
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.Container.ContainerImage = hwPDH
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{}, kc, nil
+       }
  
        err = cr.LoadImage()
        c.Check(err, NotNil)
@@@ -637,6 -652,10 +660,10 @@@ func (s *TestSuite) TestRunContainer(c 
        cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
  
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{}, kc, nil
+       }
+ 
        var logs TestLogs
        cr.NewLogWriter = logs.NewTestLoggingWriter
        cr.Container.ContainerImage = hwPDH
@@@ -780,8 -799,8 +807,8 @@@ func (s *TestSuite) fullRunHelper(c *C
                }
                return d, err
        }
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
-               return &ArvTestClient{secretMounts: secretMounts}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil
        }
  
        if extraMounts != nil && len(extraMounts) > 0 {
@@@ -1077,8 -1096,8 +1104,8 @@@ func (s *TestSuite) testStopContainer(
        cr, err := NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
-               return &ArvTestClient{}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{}, &KeepTestClient{}, nil
        }
        setup(cr)
  
@@@ -1561,8 -1580,8 +1588,8 @@@ func (s *TestSuite) stdoutErrorRunHelpe
        c.Assert(err, IsNil)
        am := &ArvMountCmdLine{}
        cr.RunArvMount = am.ArvMountTest
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
-               return &ArvTestClient{}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
+               return &ArvTestClient{}, &KeepTestClient{}, nil
        }
  
        err = cr.Run()