Merge branch '17719-keep-web-pdh-cache'

fixes #17719

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index e4ef96b194f84306b778870a36562c0b7d7b7703..0f29c543b4fbaa9fb2fbc1ef5d6a0f8adaec2eda 100644 (file)
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -189,7 +189,7 @@ GEM
     mimemagic (0.3.8)
       nokogiri (~> 1)
     mini_mime (1.0.2)
-    mini_portile2 (2.5.0)
+    mini_portile2 (2.5.1)
     minitest (5.10.3)
     mocha (1.8.0)
       metaclass (~> 0.0.1)
@@ -206,7 +206,7 @@ GEM
     net-ssh-gateway (2.0.0)
       net-ssh (>= 4.0.0)
     nio4r (2.5.7)
-    nokogiri (1.11.2)
+    nokogiri (1.11.5)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     npm-rails (0.2.1)

diff --git a/apps/workbench/app/models/arvados_resource_list.rb b/apps/workbench/app/models/arvados_resource_list.rb
index 99502bd56ed04951695e8bcb15704b64ea4b46e5..75a9429a43739f7f3c024f496d316b1d4e69cf86 100644 (file)
--- a/apps/workbench/app/models/arvados_resource_list.rb
+++ b/apps/workbench/app/models/arvados_resource_list.rb
@@ -223,6 +223,7 @@ class ArvadosResourceList
     api_params[:filters] = @filters if @filters
     api_params[:distinct] = @distinct if @distinct
     api_params[:include_trash] = @include_trash if @include_trash
+    api_params[:cluster_id] = Rails.configuration.ClusterID
     if @fetch_multiple_pages
       # Default limit to (effectively) api server's MAX_LIMIT
       api_params[:limit] = 2**(0.size*8 - 1) - 1

diff --git a/lib/controller/federation/list.go b/lib/controller/federation/list.go
index 183557eb15a4780bbe3388e3185ae9064dc609e3..039caac574e479bdad181dfeed745dd3255640cf 100644 (file)
--- a/lib/controller/federation/list.go
+++ b/lib/controller/federation/list.go
@@ -113,6 +113,11 @@ func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions
                _, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
                return err
        }
+       if opts.ClusterID != "" {
+               // Client explicitly selected cluster
+               _, err := fn(ctx, conn.cluster.ClusterID, conn.chooseBackend(opts.ClusterID), opts)
+               return err
+       }
 
        cannotSplit := false
        var matchAllFilters map[string]bool

diff --git a/lib/crunchrun/copier.go b/lib/crunchrun/copier.go
index 1b0f168b88856e8251108f11e928321b5d642c0b..132101028ea4d6d5b6b8a76df5238d7ceb0effb7 100644 (file)
--- a/lib/crunchrun/copier.go
+++ b/lib/crunchrun/copier.go
@@ -331,8 +331,8 @@ func (cp *copier) walkHostFS(dest, src string, maxSymlinks int, includeMounts bo
                })
                return nil
        }
-
-       return fmt.Errorf("Unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
+       cp.logger.Printf("Skipping unsupported file type (mode %o) in output dir: %q", fi.Mode(), src)
+       return nil
 }
 
 // Return the host path that was mounted at the given path in the

diff --git a/lib/crunchrun/copier_test.go b/lib/crunchrun/copier_test.go
index 777b715d76dd8bb57e9d5b34309ee70b356df888..07fd795efe45a75c6390520a88de39e479ca1f72 100644 (file)
--- a/lib/crunchrun/copier_test.go
+++ b/lib/crunchrun/copier_test.go
@@ -5,27 +5,31 @@
 package crunchrun
 
 import (
+       "bytes"
        "io"
        "io/ioutil"
        "os"
+       "syscall"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "git.arvados.org/arvados.git/sdk/go/arvadosclient"
        "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "github.com/sirupsen/logrus"
        check "gopkg.in/check.v1"
 )
 
 var _ = check.Suite(&copierSuite{})
 
 type copierSuite struct {
-       cp copier
+       cp  copier
+       log bytes.Buffer
 }
 
 func (s *copierSuite) SetUpTest(c *check.C) {
-       tmpdir, err := ioutil.TempDir("", "crunch-run.test.")
-       c.Assert(err, check.IsNil)
+       tmpdir := c.MkDir()
        api, err := arvadosclient.MakeArvadosClient()
        c.Assert(err, check.IsNil)
+       s.log = bytes.Buffer{}
        s.cp = copier{
                client:        arvados.NewClientFromEnv(),
                arvClient:     api,
@@ -37,13 +41,10 @@ func (s *copierSuite) SetUpTest(c *check.C) {
                secretMounts: map[string]arvados.Mount{
                        "/secret_text": {Kind: "text", Content: "xyzzy"},
                },
+               logger: &logrus.Logger{Out: &s.log, Formatter: &logrus.TextFormatter{}, Level: logrus.InfoLevel},
        }
 }
 
-func (s *copierSuite) TearDownTest(c *check.C) {
-       os.RemoveAll(s.cp.hostOutputDir)
-}
-
 func (s *copierSuite) TestEmptyOutput(c *check.C) {
        err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
        c.Check(err, check.IsNil)
@@ -59,6 +60,8 @@ func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
        _, err = io.WriteString(f, "foo")
        c.Assert(err, check.IsNil)
        c.Assert(f.Close(), check.IsNil)
+       err = syscall.Mkfifo(s.cp.hostOutputDir+"/dir1/fifo", 0644)
+       c.Assert(err, check.IsNil)
 
        err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
        c.Check(err, check.IsNil)
@@ -67,6 +70,7 @@ func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
                {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
                {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
        })
+       c.Check(s.log.String(), check.Matches, `.* msg="Skipping unsupported file type \(mode 200000644\) in output dir: \\"/ctr/outdir/dir1/fifo\\""\n`)
 }
 
 func (s *copierSuite) TestSymlinkCycle(c *check.C) {

diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 5dbdb07f2ce11c3abdb54d1fb7bd02368874e0be..58504d057a9a085f9afbc7a24dff161d8f96b340 100644 (file)
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -154,7 +154,7 @@ GEM
     mimemagic (0.3.8)
       nokogiri (~> 1)
     mini_mime (1.0.2)
-    mini_portile2 (2.5.0)
+    mini_portile2 (2.5.1)
     minitest (5.10.3)
     mocha (1.8.0)
       metaclass (~> 0.0.1)
@@ -170,7 +170,7 @@ GEM
     net-ssh-gateway (2.0.0)
       net-ssh (>= 4.0.0)
     nio4r (2.5.7)
-    nokogiri (1.11.2)
+    nokogiri (1.11.5)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     oauth2 (1.4.1)