[User, Group].each do |type|
type
.filter([['uuid','in',@share_links.collect(&:tail_uuid)]])
+ .with_count("none")
+ .fetch_multiple_pages(false)
.each do |o|
uuid_map[o.uuid] = o
end
client_max_body_size 0;
proxy_http_version 1.1;
proxy_request_buffering off;
+ proxy_max_temp_file_size 0;
}
}
</pre></notextile>
proxy_set_header X-Real-IP $remote_addr;
proxy_http_version 1.1;
proxy_request_buffering off;
+ proxy_max_temp_file_size 0;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Upgrading to CWL v1.1
+h2(#v12). Upgrading your workflows to CWL v1.2
+
+If you are starting from a CWL v1.0 document, see "Upgrading your workflows to CWL v1.1":#v11 below.
+
+If you are starting from a CWL v1.1 document, you should be able to trivially change @cwlVersion: v1.1@ to @cwlVersion: v1.2@ to be able to take advantage of the new features of v1.2, such as conditional workflow steps.
+
+h2(#v11). Upgrading your workflows to CWL v1.1
CWL v1.1 introduces several features to the standard that were previously available as Arvados extensions. CWL v1.1 syntax is backwards compatible with v1.0, so you can just change @cwlVersion: v1.0@ to @cwlVersion: v1.1@ and update your script to using the standard features. On Arvados, there is only one behavior change between CWL v1.0 and v1.1 to be aware of: for performance reasons, Directory listings are no longer loaded by default. To control loading Directory listings, use "loadListing":https://www.commonwl.org/v1.1/CommandLineTool.html#CommandInputParameter or "LoadListingRequirement":https://www.commonwl.org/v1.1/CommandLineTool.html#LoadListingRequirement (the extension @cwltool:LoadListingRequirement@ is deprecated.)
content = []byte(fmtKeepproxyConfig("", false))
cluster, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.IsNil)
c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
cfg, err := testLoader(c, yaml, nil).Load()
c.Assert(err, check.IsNil)
cc, err := cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
_, hasSample := cc.InstanceTypes["SAMPLE"]
c.Check(hasSample, check.Equals, false)
if strings.Contains(yaml, "Foo") {
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.ContainerList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.SpecimenList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.UserList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.CollectionList(ctx, options)
if err != nil {
return nil, err
// backend.
func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions, fn func(context.Context, string, arvados.API, arvados.ListOptions) ([]string, error)) error {
- if opts.BypassFederation {
+ if opts.BypassFederation || opts.ForwardedFor != "" {
// Client requested no federation. Pass through.
_, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
return err
done, err := fn(ctx, clusterID, backend, remoteOpts)
if err != nil {
- errs <- httpErrorf(http.StatusBadGateway, err.Error())
+ errs <- httpErrorf(http.StatusBadGateway, "%s", err.Error())
return
}
progress := false
resp, err = arvados.InsecureHTTPClient.Do(req)
if c.Check(err, check.IsNil) {
err = json.NewDecoder(resp.Body).Decode(&cr)
+ c.Check(err, check.IsNil)
c.Check(cr.UUID, check.Matches, "z2222-.*")
}
}
c.Assert(err, check.IsNil)
ln, err := net.Listen("tcp", "127.0.0.1:0")
+ c.Assert(err, check.IsNil)
s.ldap = &godap.LDAPServer{
Listener: ln,
Handlers: []godap.LDAPRequestHandler{
c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.HasLen, 27)
c.Check(jresp["state"], check.Equals, "Locked")
- _, rr, jresp = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
+ _, rr, _ = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
c.Check(rr.Code, check.Equals, http.StatusUnprocessableEntity)
c.Check(rr.Body.String(), check.Not(check.Matches), `.*"uuid":.*`)
_, rr, jresp = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
c.Check(sp.Properties["foo"], check.Equals, "bar")
spGet, err := s.conn.SpecimenGet(s.ctx, arvados.GetOptions{UUID: sp.UUID})
+ c.Check(err, check.IsNil)
c.Check(spGet.UUID, check.Equals, sp.UUID)
c.Check(spGet.Properties["foo"], check.Equals, "bar")
spList, err := s.conn.SpecimenList(s.ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(err, check.IsNil)
c.Check(spList.ItemsAvailable, check.Equals, 1)
c.Assert(spList.Items, check.HasLen, 1)
c.Check(spList.Items[0].UUID, check.Equals, sp.UUID)
anonCtx := context.WithValue(context.Background(), contextKeyTestTokens, []string{arvadostest.AnonymousToken})
spList, err = s.conn.SpecimenList(anonCtx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(err, check.IsNil)
c.Check(spList.ItemsAvailable, check.Equals, 0)
c.Check(spList.Items, check.HasLen, 0)
spDel, err := s.conn.SpecimenDelete(s.ctx, arvados.DeleteOptions{UUID: sp.UUID})
+ c.Check(err, check.IsNil)
c.Check(spDel.UUID, check.Equals, sp.UUID)
}
} else if arg == "debug" {
logger.SetLevel(logrus.DebugLevel)
} else {
- logger.Warnf("unkown option: %s\n", arg)
+ logger.Warnf("unknown option: %s\n", arg)
}
}
if hostname == "" || hostname == "-" {
c.Assert(err, check.IsNil)
t := time.Now().Add(-time.Hour * 24 * 365)
err = os.Chtimes(trashfile, t, t)
+ c.Assert(err, check.IsNil)
}
var stdout, stderr bytes.Buffer
continue
}
body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
c.Logf("status %d, body %s", resp.StatusCode, string(body))
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
break
def add_arv_hints():
cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")
cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
- res10 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
- res11 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
- customschema10 = res10.read().decode('utf-8')
- customschema11 = res11.read().decode('utf-8')
- use_custom_schema("v1.0", "http://arvados.org/cwl", customschema10)
- use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema11)
- use_custom_schema("v1.1", "http://arvados.org/cwl", customschema11)
- res10.close()
- res11.close()
+ supported_versions = ["v1.0", "v1.1", "v1.2"]
+ for s in supported_versions:
+ res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-%s.yml' % s)
+ customschema = res.read().decode('utf-8')
+ use_custom_schema(s, "http://arvados.org/cwl", customschema)
+ res.close()
cwltool.process.supportedProcessRequirements.extend([
"http://arvados.org/cwl#RunInSingleContainer",
"http://arvados.org/cwl#OutputDirType",
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+$base: "http://arvados.org/cwl#"
+$namespaces:
+ cwl: "https://w3id.org/cwl/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+$graph:
+- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
+
+- name: cwltool:Secrets
+ type: record
+ inVocab: false
+ extends: cwl:ProcessRequirement
+ fields:
+ class:
+ type: string
+ doc: "Always 'Secrets'"
+ jsonldPredicate:
+ "_id": "@type"
+ "_type": "@vocab"
+ secrets:
+ type: string[]
+ doc: |
+ List one or more input parameters that are sensitive (such as passwords)
+ which will be deliberately obscured from logging.
+ jsonldPredicate:
+ "_type": "@id"
+ refScope: 0
+
+- name: RunInSingleContainer
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Indicates that a subworkflow should run in a single container
+ and not be scheduled as separate steps.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:RunInSingleContainer'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+
+- name: OutputDirType
+ type: enum
+ symbols:
+ - local_output_dir
+ - keep_output_dir
+ doc:
+ - |
+ local_output_dir: Use regular file system local to the compute node.
+ There must be sufficient local scratch space to store entire output;
+ specify this with `outdirMin` of `ResourceRequirement`. Files are
+ batch uploaded to Keep when the process completes. Most compatible, but
+ upload step can be time consuming for very large files.
+ - |
+ keep_output_dir: Use writable Keep mount. Files are streamed to Keep as
+ they are written. Does not consume local scratch space, but does consume
+ RAM for output buffers (up to 192 MiB per file simultaneously open for
+ writing.) Best suited to processes which produce sequential output of
+ large files (non-sequential writes may produced fragmented file
+ manifests). Supports regular files and directories, does not support
+ special files such as symlinks, hard links, named pipes, named sockets,
+ or device nodes.
+
+
+- name: RuntimeConstraints
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Set Arvados-specific runtime hints.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:RuntimeConstraints'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ - name: keep_cache
+ type: int?
+ doc: |
+ Size of file data buffer for Keep mount in MiB. Default is 256
+ MiB. Increase this to reduce cache thrashing in situations such as
+ accessing multiple large (64+ MiB) files at the same time, or
+ performing random access on a large file.
+ - name: outputDirType
+ type: OutputDirType?
+ doc: |
+ Preferred backing store for output staging. If not specified, the
+ system may choose which one to use.
+
+- name: PartitionRequirement
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Select preferred compute partitions on which to run jobs.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:PartitionRequirement'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ - name: partition
+ type:
+ - string
+ - string[]
+
+- name: APIRequirement
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Indicates that process wants to access to the Arvados API. Will be granted
+ limited network access and have ARVADOS_API_HOST and ARVADOS_API_TOKEN set
+ in the environment.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:APIRequirement'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+
+- name: IntermediateOutput
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify desired handling of intermediate output collections.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:IntermediateOutput'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ outputTTL:
+ type: int
+ doc: |
+ If the value is greater than zero, consider intermediate output
+ collections to be temporary and should be automatically
+ trashed. Temporary collections will be trashed `outputTTL` seconds
+ after creation. A value of zero means intermediate output should be
+ retained indefinitely (this is the default behavior).
+
+ Note: arvados-cwl-runner currently does not take workflow dependencies
+ into account when setting the TTL on an intermediate output
+ collection. If the TTL is too short, it is possible for a collection to
+ be trashed before downstream steps that consume it are started. The
+ recommended minimum value for TTL is the expected duration of the
+ entire the workflow.
+
+- name: WorkflowRunnerResources
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify memory or cores resource request for the CWL runner process itself.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:WorkflowRunnerResources'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ ramMin:
+ type: int?
+ doc: Minimum RAM, in mebibytes (2**20)
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/ramMin"
+ coresMin:
+ type: int?
+ doc: Minimum cores allocated to cwl-runner
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
+ keep_cache:
+ type: int?
+ doc: |
+ Size of collection metadata cache for the workflow runner, in
+ MiB. Default 256 MiB. Will be added on to the RAM request
+ when determining node size to request.
+ jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+
+- name: ClusterTarget
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify where a workflow step should run
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:ClusterTarget'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ cluster_id:
+ type: string?
+ doc: The cluster to run the container
+ project_uuid:
+ type: string?
+ doc: The project that will own the container requests and intermediate collections
download_url="https://github.com/arvados/arvados.git",
license='Apache 2.0',
packages=find_packages(),
- package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml']},
+ package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml', 'arv-cwl-schema-v1.2.yml']},
scripts=[
'bin/cwl-runner',
'bin/arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200720165847',
+ 'cwltool==3.0.20200807132242',
'schema-salad==7.0.20200612160654',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
IncludeTrash bool `json:"include_trash"`
IncludeOldVersions bool `json:"include_old_versions"`
BypassFederation bool `json:"bypass_federation"`
+ ForwardedFor string `json:"forwarded_for,omitempty"`
}
type CreateOptions struct {
c.Check(err, check.IsNil)
c.Check(d.D, check.Equals, Duration(time.Second+234*time.Millisecond))
buf, err := json.Marshal(d)
+ c.Check(err, check.IsNil)
c.Check(string(buf), check.Equals, `{"D":"1.234s"}`)
for _, trial := range []struct {
f, err := s.fs.Open(path)
c.Assert(err, check.IsNil)
fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
c.Check(len(fis), check.Not(check.Equals), 0)
ok := false
f, err = s.fs.Open("/by_id/" + path)
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
+ c.Assert(err, check.IsNil)
var names []string
for _, fi := range fis {
names = append(names, fi.Name())
f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
+ c.Assert(err, check.IsNil)
var names []string
for _, fi := range fis {
names = append(names, fi.Name())
func (s *ServerRequiredSuite) TestGetInvalidUUID(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("collections", "", nil, &getback)
func (s *ServerRequiredSuite) TestGetValidUUID(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("collections", "zzzzz-4zz18-abcdeabcdeabcde", nil, &getback)
func (s *ServerRequiredSuite) TestInvalidResourceType(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultCollectionReplication(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
value, err := arv.Discovery("defaultCollectionReplication")
c.Assert(err, IsNil)
c.Assert(value, NotNil)
func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
value, err := arv.Discovery("noSuchParameter")
c.Assert(err, NotNil)
c.Assert(value, IsNil)
gotReq := make(map[string]interface{})
err = dec.Decode(&gotReq)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotReq)
c.Check(gotReq["RequestID"], check.Matches, "req-[a-z0-9]{20}")
c.Check(gotReq["reqForwardedFor"], check.Equals, "1.2.3.4:12345")
gotResp := make(map[string]interface{})
err = dec.Decode(&gotResp)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotResp)
c.Check(gotResp["RequestID"], check.Equals, gotReq["RequestID"])
c.Check(gotResp["reqForwardedFor"], check.Equals, "1.2.3.4:12345")
gotReq := make(map[string]interface{})
err = dec.Decode(&gotReq)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotReq)
gotResp := make(map[string]interface{})
err = dec.Decode(&gotResp)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotResp)
if trial.expectLog {
c.Check(gotResp["respBody"], check.Equals, trial.expectBody, comment)
c.Check(err, check.Equals, want)
case string:
buf := make([]byte, len(want))
- n, err := io.ReadFull(rdr, buf)
+ _, err := io.ReadFull(rdr, buf)
c.Check(err, check.IsNil)
for i := 0; i < 4; i++ {
c.Check(string(buf), check.Equals, want)
- n, err = rdr.Read(buf)
+ n, err := rdr.Read(buf)
c.Check(n, check.Equals, 0)
c.Check(err, check.Equals, io.EOF)
}
filesize := 0
for i := range locs {
_, err := rand.Read(buf[:i])
+ c.Assert(err, check.IsNil)
h.Write(buf[:i])
locs[i], _, err = s.kc.PutB(buf[:i])
c.Assert(err, check.IsNil)
offset := rand.Intn(len(buf) - 1)
count := rand.Intn(len(buf) - offset)
if rand.Intn(2) == 0 {
- curPos, err = rdr.Seek(int64(offset)-curPos, io.SeekCurrent)
+ curPos, _ = rdr.Seek(int64(offset)-curPos, io.SeekCurrent)
} else {
- curPos, err = rdr.Seek(int64(offset), io.SeekStart)
+ curPos, _ = rdr.Seek(int64(offset), io.SeekStart)
}
c.Check(curPos, check.Equals, int64(offset))
for count > 0 {
count -= n
}
curPos, err = rdr.Seek(0, io.SeekCurrent)
+ c.Check(err, check.IsNil)
c.Check(curPos, check.Equals, int64(offset))
}
c.Check(md5.Sum(buf), check.DeepEquals, md5.Sum(testdata))
c.Assert(err, Equals, nil)
kc, err := MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 2)
arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 3)
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
make(chan string, 4)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ksGateway.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, n, _, err := kc.Get(barhash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, nil)
<-st.handled
r, n, _, err = kc.Get(foohash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, BadChecksum)
content}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 3
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
# SPDX-License-Identifier: Apache-2.0
import arvados
+import arvados.errors
import json
import sys
users = apiC.users().list().execute()
check_A(users)
+
+####
+# bug 16683 tests
+
+# Check that this query returns empty, instead of returning a 500 or
+# 502 error.
+# Yes, we're asking for a group from the users endpoint. This is not a
+# mistake, this is something workbench does to populate the sharing
+# dialog.
+clusterID_B = apiB.configs().get().execute()["ClusterID"]
+i = apiB.users().list(filters=[["uuid", "in", ["%s-j7d0g-fffffffffffffff" % clusterID_B]]], count="none").execute()
+assert len(i["items"]) == 0
+
+# Check that we can create a project and give a remote user access to it
+
+tok3 = apiA.api_client_authorizations().create(body={"api_client_authorization": {"owner_uuid": by_username["case3"]}}).execute()
+tok4 = apiA.api_client_authorizations().create(body={"api_client_authorization": {"owner_uuid": by_username["case4"]}}).execute()
+
+v2_token3 = "v2/%s/%s" % (tok3["uuid"], tok3["api_token"])
+v2_token4 = "v2/%s/%s" % (tok4["uuid"], tok4["api_token"])
+
+apiB_3 = arvados.api(host=j["arvados_api_hosts"][1], token=v2_token3, insecure=True)
+apiB_4 = arvados.api(host=j["arvados_api_hosts"][1], token=v2_token4, insecure=True)
+
+assert apiB_3.users().current().execute()["uuid"] == by_username["case3"]
+assert apiB_4.users().current().execute()["uuid"] == by_username["case4"]
+
+newproject = apiB_3.groups().create(body={"group_class": "project",
+ "name":"fed test project"},
+ ensure_unique_name=True).execute()
+
+try:
+ # Expect to fail
+ apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+except arvados.errors.ApiError as e:
+ if e.resp['status'] == '404':
+ pass
+ else:
+ raise
+
+l = apiB_3.links().create(body={"link_class": "permission",
+ "name":"can_read",
+ "tail_uuid": by_username["case4"],
+ "head_uuid": newproject["uuid"]}).execute()
+
+# Expect to succeed
+apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+
+# remove permission
+apiB_3.links().delete(uuid=l["uuid"]).execute()
+
+try:
+ # Expect to fail again
+ apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+except arvados.errors.ApiError as e:
+ if e.resp['status'] == '404':
+ pass
+ else:
+ raise
+
print("Passed checks")
%r/[a-z0-9]{5}-#{uuid_prefix}-[a-z0-9]{15}/
end
+ def check_readable_uuid attr, attr_value
+ return if attr_value.nil?
+ if (r = ArvadosModel::resource_class_for_uuid attr_value)
+ unless skip_uuid_read_permission_check.include? attr
+ r = r.readable_by(current_user)
+ end
+ if r.where(uuid: attr_value).count == 0
+ errors.add(attr, "'#{attr_value}' not found")
+ end
+ else
+ # Not a valid uuid or PDH, but that (currently) is not an error.
+ end
+ end
+
def ensure_valid_uuids
specials = [system_user_uuid]
next if skip_uuid_existence_check.include? attr
attr_value = send attr
next if specials.include? attr_value
- if attr_value
- if (r = ArvadosModel::resource_class_for_uuid attr_value)
- unless skip_uuid_read_permission_check.include? attr
- r = r.readable_by(current_user)
- end
- if r.where(uuid: attr_value).count == 0
- errors.add(attr, "'#{attr_value}' not found")
- end
- end
- end
+ check_readable_uuid attr, attr_value
end
end
end
protected
+ def check_readable_uuid attr, attr_value
+ if attr == 'tail_uuid' &&
+ !attr_value.nil? &&
+ self.link_class == 'permission' &&
+ attr_value[0..4] != Rails.configuration.ClusterID &&
+ ApiClientAuthorization.remote_host(uuid_prefix: attr_value[0..4]) &&
+ ArvadosModel::resource_class_for_uuid(attr_value) == User
+ # Permission link tail is a remote user (the user permissions
+ # are being granted to), so bypass the standard check that a
+ # referenced object uuid is readable by current user.
+ #
+ # We could do a call to the remote cluster to check if the user
+ # in tail_uuid exists. This would detect copy-and-paste errors,
+ # but add another way for the request to fail, and I don't think
+ # it would improve security. It doesn't seem to be worth the
+ # complexity tradeoff.
+ true
+ else
+ super
+ end
+ end
+
def permission_to_attach_to_objects
# Anonymous users cannot write links
return false if !current_user
head_obj = ArvadosModel.find_by_uuid(head_uuid)
+ if head_obj.nil?
+ errors.add(:head_uuid, "does not exist")
+ return false
+ end
+
# No permission links can be pointed to past collection versions
if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
errors.add(:head_uuid, "cannot point to a past version of a collection")
users(:active).uuid.sub(/-\w+$/, "-#{'z' * 15}"))
end
+ test "link granting permission to remote user is valid" do
+ refute new_active_link_valid?(tail_uuid:
+ users(:active).uuid.sub(/^\w+-/, "foooo-"))
+ Rails.configuration.RemoteClusters = Rails.configuration.RemoteClusters.merge({foooo: ActiveSupport::InheritableOptions.new({Host: "bar.com"})})
+ assert new_active_link_valid?(tail_uuid:
+ users(:active).uuid.sub(/^\w+-/, "foooo-"))
+ end
+
test "link granting non-project permission to unreadable user is invalid" do
refute new_active_link_valid?(tail_uuid: users(:admin).uuid,
head_uuid: collections(:bar_file).uuid)
c.Fatal(err)
}
_, p, err := net.SplitHostPort(ln.Addr().String())
+ c.Check(err, check.IsNil)
ln.Close()
config := "Clusters:\n zzzzz:\n ManagementToken: abcdefg\n Services: {Keepbalance: {InternalURLs: {'http://localhost:" + p + "/': {}}}}\n"
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err := v.check(s.metadata.URL + "/latest")
+ c.Check(err, check.IsNil)
creds, err := v.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
+ c.Check(err, check.IsNil)
c.Check(creds.AccessKeyID, check.Equals, "ASIAIOSFODNN7EXAMPLE")
c.Check(creds.SecretAccessKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err = deadv.check(s.metadata.URL + "/latest")
+ c.Check(err, check.IsNil)
_, err = deadv.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
c.Check(err, check.ErrorMatches, `(?s).*EC2RoleRequestError: no EC2 instance role found.*`)
c.Check(err, check.ErrorMatches, `(?s).*404.*`)
projects / arvados.git / commitdiff