From: Peter Amstutz Date: Mon, 14 Feb 2022 18:50:05 +0000 (-0500) Subject: Merge branch '18723-cwl-upload' refs #18723 X-Git-Tag: 2.4.0~90 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/4475c4d80d00615a8feb24e3483a43f19736b373?hp=e2b3ed63a292537ec7bf37a904ad2925d6afa0bf Merge branch '18723-cwl-upload' refs #18723 Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go index af0d49c80e..4fa3f26ab5 100644 --- a/lib/crunchrun/crunchrun.go +++ b/lib/crunchrun/crunchrun.go @@ -453,8 +453,8 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) { sort.Strings(binds) for _, bind := range binds { - mnt, ok := runner.Container.Mounts[bind] - if !ok { + mnt, notSecret := runner.Container.Mounts[bind] + if !notSecret { mnt = runner.SecretMounts[bind] } if bind == "stdout" || bind == "stderr" { @@ -523,8 +523,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) { } } else { src = fmt.Sprintf("%s/tmp%d", runner.ArvMountPoint, tmpcount) - arvMountCmd = append(arvMountCmd, "--mount-tmp") - arvMountCmd = append(arvMountCmd, fmt.Sprintf("tmp%d", tmpcount)) + arvMountCmd = append(arvMountCmd, "--mount-tmp", fmt.Sprintf("tmp%d", tmpcount)) tmpcount++ } if mnt.Writable { @@ -584,9 +583,32 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) { if err != nil { return nil, fmt.Errorf("writing temp file: %v", err) } - if strings.HasPrefix(bind, runner.Container.OutputPath+"/") { + if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && (notSecret || runner.Container.Mounts[runner.Container.OutputPath].Kind != "collection") { + // In most cases, if the container + // specifies a literal file inside the + // output path, we copy it into the + // output directory (either a mounted + // collection or a staging area on the + // host fs). If it's a secret, it will + // be skipped when copying output from + // staging to Keep later. copyFiles = append(copyFiles, copyFile{tmpfn, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]}) } else { + // If a secret is outside OutputPath, + // we bind mount the secret file + // directly just like other mounts. We + // also use this strategy when a + // secret is inside OutputPath but + // OutputPath is a live collection, to + // avoid writing the secret to + // Keep. Attempting to remove a + // bind-mounted secret file from + // inside the container will return a + // "Device or resource busy" error + // that might not be handled well by + // the container, which is why we + // don't use this strategy when + // OutputPath is a staging directory. bindmounts[bind] = bindmount{HostPath: tmpfn, ReadOnly: true} } diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go index 2ec35f0559..806a83eef9 100644 --- a/lib/crunchrun/crunchrun_test.go +++ b/lib/crunchrun/crunchrun_test.go @@ -44,6 +44,7 @@ type TestSuite struct { runner *ContainerRunner executor *stubExecutor keepmount string + keepmountTmp []string testDispatcherKeepClient KeepTestClient testContainerKeepClient KeepTestClient } @@ -62,10 +63,20 @@ func (s *TestSuite) SetUpTest(c *C) { } s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) { s.runner.ArvMountPoint = s.keepmount + for i, opt := range cmd { + if opt == "--mount-tmp" { + err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700) + if err != nil { + return nil, err + } + s.keepmountTmp = append(s.keepmountTmp, cmd[i+1]) + } + } return nil, nil } s.keepmount = c.MkDir() err = os.Mkdir(s.keepmount+"/by_id", 0755) + s.keepmountTmp = nil c.Assert(err, IsNil) err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755) c.Assert(err, IsNil) @@ -638,8 +649,6 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi s.runner.statInterval = 100 * time.Millisecond s.runner.containerWatchdogInterval = time.Second - am := &ArvMountCmdLine{} - s.runner.RunArvMount = am.ArvMountTest realTemp := c.MkDir() tempcount := 0 @@ -2007,6 +2016,44 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) { c.Check(s.api.CalledWith("container.state", "Complete"), NotNil) c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil) c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil) + + // under secret mounts, output dir is a collection, not captured in output + helperRecord = `{ + "command": ["true"], + "container_image": "` + arvadostest.DockerImage112PDH + `", + "cwd": "/bin", + "mounts": { + "/tmp": {"kind": "collection", "writable": true} + }, + "secret_mounts": { + "/tmp/secret.conf": {"kind": "text", "content": "mypassword"} + }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {}, + "state": "Locked" + }` + + s.SetUpTest(c) + _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() { + // secret.conf should be provisioned as a separate + // bind mount, i.e., it should not appear in the + // (fake) fuse filesystem as viewed from the host. + content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf") + if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) { + c.Logf("secret.conf: content %q, err %#v", content, err) + } + err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700) + c.Check(err, IsNil) + }) + + content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text") + c.Check(err, IsNil) + c.Check(string(content), Equals, "mypassword") + c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true}) + c.Check(s.api.CalledWith("container.exit_code", 0), NotNil) + c.Check(s.api.CalledWith("container.state", "Complete"), NotNil) + c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil) } type FakeProcess struct { diff --git a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java index 4cd08b7832..0145b0fc4c 100644 --- a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java +++ b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java @@ -26,6 +26,15 @@ public class KeepWebApiClient extends BaseApiClient { return newFileCall(request); } + public String delete(String collectionUuid, String filePathName) { + Request request = getRequestBuilder() + .url(getUrlBuilder(collectionUuid, filePathName).build()) + .delete() + .build(); + + return newCall(request); + } + private HttpUrl.Builder getUrlBuilder(String collectionUuid, String filePathName) { return new HttpUrl.Builder() .scheme(config.getApiProtocol()) diff --git a/sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/ListArgument.java b/sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/ListArgument.java index 70231e6766..ca86c585e8 100644 --- a/sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/ListArgument.java +++ b/sdk/java-v2/src/main/java/org/arvados/client/api/model/argument/ListArgument.java @@ -14,7 +14,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.List; @JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ "limit", "offset", "filters", "order", "select", "distinct", "count" }) +@JsonPropertyOrder({ "limit", "offset", "filters", "order", "select", "distinct", "count", "exclude_home_project" }) public class ListArgument extends Argument { @JsonProperty("limit") @@ -38,8 +38,10 @@ public class ListArgument extends Argument { @JsonProperty("count") private Count count; + @JsonProperty("exclude_home_project") + private Boolean excludeHomeProject; - ListArgument(Integer limit, Integer offset, List filters, List order, List select, Boolean distinct, Count count) { + ListArgument(Integer limit, Integer offset, List filters, List order, List select, Boolean distinct, Count count, Boolean excludeHomeProject) { this.limit = limit; this.offset = offset; this.filters = filters; @@ -47,6 +49,7 @@ public class ListArgument extends Argument { this.select = select; this.distinct = distinct; this.count = count; + this.excludeHomeProject = excludeHomeProject; } public static ListArgumentBuilder builder() { @@ -70,6 +73,7 @@ public class ListArgument extends Argument { private List select; private Boolean distinct; private Count count; + private Boolean excludeHomeProject; ListArgumentBuilder() { } @@ -109,15 +113,21 @@ public class ListArgument extends Argument { return this; } + public ListArgument.ListArgumentBuilder excludeHomeProject(Boolean excludeHomeProject) { + this.excludeHomeProject = excludeHomeProject; + return this; + } + public ListArgument build() { - return new ListArgument(limit, offset, filters, order, select, distinct, count); + return new ListArgument(limit, offset, filters, order, select, distinct, count, excludeHomeProject); } public String toString() { return "ListArgument.ListArgumentBuilder(limit=" + this.limit + ", offset=" + this.offset + ", filters=" + this.filters + ", order=" + this.order + ", select=" + this.select + - ", distinct=" + this.distinct + ", count=" + this.count + ")"; + ", distinct=" + this.distinct + ", count=" + this.count + + ", excludeHomeProject=" + this.excludeHomeProject + ")"; } } }