[User, Group].each do |type|
type
.filter([['uuid','in',@share_links.collect(&:tail_uuid)]])
+ .with_count("none")
+ .fetch_multiple_pages(false)
.each do |o|
uuid_map[o.uuid] = o
end
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-What you need to know and do in order to upgrade your Arvados installation.
+For Arvados administrators, this page will cover what you need to know and do in order to ensure a smooth upgrade of your Arvados installation. For general release notes covering features added and bugs fixed, see "Arvados releases":https://arvados.org/releases .
h2. General process
# Consult upgrade notes below to see if any manual configuration updates are necessary.
# Wait for the cluster to be idle and stop Arvados services.
# Install new packages using @apt-get upgrade@ or @yum upgrade@.
-# Package installation scripts will perform any necessary data migrations.
+# Wait for package installation scripts as they perform any necessary data migrations.
# Restart Arvados services.
h2. Upgrade notes
client_max_body_size 0;
proxy_http_version 1.1;
proxy_request_buffering off;
+ proxy_max_temp_file_size 0;
}
}
</pre></notextile>
proxy_set_header X-Real-IP $remote_addr;
proxy_http_version 1.1;
proxy_request_buffering off;
+ proxy_max_temp_file_size 0;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Upgrading to CWL v1.1
+h2(#v12). Upgrading your workflows to CWL v1.2
+
+If you are starting from a CWL v1.0 document, see "Upgrading your workflows to CWL v1.1":#v11 below.
+
+If you are starting from a CWL v1.1 document, you should be able to trivially change @cwlVersion: v1.1@ to @cwlVersion: v1.2@ to be able to take advantage of the new features of v1.2, such as conditional workflow steps.
+
+h2(#v11). Upgrading your workflows to CWL v1.1
CWL v1.1 introduces several features to the standard that were previously available as Arvados extensions. CWL v1.1 syntax is backwards compatible with v1.0, so you can just change @cwlVersion: v1.0@ to @cwlVersion: v1.1@ and update your script to using the standard features. On Arvados, there is only one behavior change between CWL v1.0 and v1.1 to be aware of: for performance reasons, Directory listings are no longer loaded by default. To control loading Directory listings, use "loadListing":https://www.commonwl.org/v1.1/CommandLineTool.html#CommandInputParameter or "LoadListingRequirement":https://www.commonwl.org/v1.1/CommandLineTool.html#LoadListingRequirement (the extension @cwltool:LoadListingRequirement@ is deprecated.)
content = []byte(fmtKeepproxyConfig("", false))
cluster, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.IsNil)
c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
cfg, err := testLoader(c, yaml, nil).Load()
c.Assert(err, check.IsNil)
cc, err := cfg.GetCluster("z1111")
+ c.Assert(err, check.IsNil)
_, hasSample := cc.InstanceTypes["SAMPLE"]
c.Check(hasSample, check.Equals, false)
if strings.Contains(yaml, "Foo") {
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.ContainerList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.SpecimenList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.UserList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.CollectionList(ctx, options)
if err != nil {
return nil, err
// backend.
func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions, fn func(context.Context, string, arvados.API, arvados.ListOptions) ([]string, error)) error {
- if opts.BypassFederation {
+ if opts.BypassFederation || opts.ForwardedFor != "" {
// Client requested no federation. Pass through.
_, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
return err
done, err := fn(ctx, clusterID, backend, remoteOpts)
if err != nil {
- errs <- httpErrorf(http.StatusBadGateway, err.Error())
+ errs <- httpErrorf(http.StatusBadGateway, "%s", err.Error())
return
}
progress := false
resp, err = arvados.InsecureHTTPClient.Do(req)
if c.Check(err, check.IsNil) {
err = json.NewDecoder(resp.Body).Decode(&cr)
+ c.Check(err, check.IsNil)
c.Check(cr.UUID, check.Matches, "z2222-.*")
}
}
c.Assert(err, check.IsNil)
ln, err := net.Listen("tcp", "127.0.0.1:0")
+ c.Assert(err, check.IsNil)
s.ldap = &godap.LDAPServer{
Listener: ln,
Handlers: []godap.LDAPRequestHandler{
c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.HasLen, 27)
c.Check(jresp["state"], check.Equals, "Locked")
- _, rr, jresp = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
+ _, rr, _ = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
c.Check(rr.Code, check.Equals, http.StatusUnprocessableEntity)
c.Check(rr.Body.String(), check.Not(check.Matches), `.*"uuid":.*`)
_, rr, jresp = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
c.Check(sp.Properties["foo"], check.Equals, "bar")
spGet, err := s.conn.SpecimenGet(s.ctx, arvados.GetOptions{UUID: sp.UUID})
+ c.Check(err, check.IsNil)
c.Check(spGet.UUID, check.Equals, sp.UUID)
c.Check(spGet.Properties["foo"], check.Equals, "bar")
spList, err := s.conn.SpecimenList(s.ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(err, check.IsNil)
c.Check(spList.ItemsAvailable, check.Equals, 1)
c.Assert(spList.Items, check.HasLen, 1)
c.Check(spList.Items[0].UUID, check.Equals, sp.UUID)
anonCtx := context.WithValue(context.Background(), contextKeyTestTokens, []string{arvadostest.AnonymousToken})
spList, err = s.conn.SpecimenList(anonCtx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(err, check.IsNil)
c.Check(spList.ItemsAvailable, check.Equals, 0)
c.Check(spList.Items, check.HasLen, 0)
spDel, err := s.conn.SpecimenDelete(s.ctx, arvados.DeleteOptions{UUID: sp.UUID})
+ c.Check(err, check.IsNil)
c.Check(spDel.UUID, check.Equals, sp.UUID)
}
notify <-chan struct{}
unalloc map[arvados.InstanceType]int // idle+booting+unknown
idle map[arvados.InstanceType]int
+ unknown map[arvados.InstanceType]int
running map[string]time.Time
atQuota bool
canCreate int
defer p.Unlock()
r := map[arvados.InstanceType]int{}
for it, n := range p.unalloc {
- r[it] = n
+ r[it] = n - p.unknown[it]
}
return r
}
worker.StateBooting: len(p.unalloc) - len(p.idle),
worker.StateIdle: len(p.idle),
worker.StateRunning: len(p.running),
+ worker.StateUnknown: len(p.unknown),
}
}
func (p *stubPool) StartContainer(it arvados.InstanceType, ctr arvados.Container) bool {
"fmt"
"git.arvados.org/arvados.git/lib/dispatchcloud/container"
+ "git.arvados.org/arvados.git/lib/dispatchcloud/worker"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/sirupsen/logrus"
)
// Running containers whose crunch-run processes have exited are
// cancelled.
func (sch *Scheduler) sync() {
+ anyUnknownWorkers := sch.pool.CountWorkers()[worker.StateUnknown] > 0
running := sch.pool.Running()
qEntries, qUpdated := sch.queue.Entries()
for uuid, ent := range qEntries {
switch ent.Container.State {
case arvados.ContainerStateRunning:
if !running {
- go sch.cancel(uuid, "not running on any worker")
+ if !anyUnknownWorkers {
+ go sch.cancel(uuid, "not running on any worker")
+ }
} else if !exited.IsZero() && qUpdated.After(exited) {
go sch.cancel(uuid, "state=Running after crunch-run exited")
} else if ent.Container.Priority == 0 {
ents, _ = queue.Entries()
c.Check(ents, check.HasLen, 0)
}
+
+func (*SchedulerSuite) TestCancelOrphanedContainers(c *check.C) {
+ ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c))
+ pool := stubPool{
+ unalloc: map[arvados.InstanceType]int{test.InstanceType(1): 1},
+ unknown: map[arvados.InstanceType]int{test.InstanceType(1): 1},
+ }
+ queue := test.Queue{
+ ChooseType: chooseType,
+ Containers: []arvados.Container{
+ {
+ UUID: test.ContainerUUID(1),
+ Priority: 0,
+ State: arvados.ContainerStateRunning,
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 1,
+ RAM: 1 << 30,
+ },
+ },
+ },
+ }
+ queue.Update()
+
+ ents, _ := queue.Entries()
+ c.Check(ents, check.HasLen, 1)
+
+ sch := New(ctx, &queue, &pool, time.Millisecond, time.Millisecond)
+
+ // Sync shouldn't cancel the container because it might be
+ // running on the VM with state=="unknown".
+ //
+ // (Cancel+forget happens asynchronously and requires multiple
+ // sync() calls, so even after 10x sync-and-sleep iterations,
+ // we aren't 100% confident that sync isn't trying to
+ // cancel. But in the test environment, the goroutines started
+ // by sync() access stubs and therefore run quickly, so it
+ // works fine in practice. We accept that if the code is
+ // broken, the test will still pass occasionally.)
+ for i := 0; i < 10; i++ {
+ sch.sync()
+ time.Sleep(time.Millisecond)
+ }
+ ents, _ = queue.Entries()
+ c.Check(ents, check.HasLen, 1)
+ c.Check(ents[test.ContainerUUID(1)].Container.State, check.Equals, arvados.ContainerStateRunning)
+
+ // Sync should cancel & forget the container when the
+ // "unknown" node goes away.
+ //
+ // (As above, cancel+forget is async and requires multiple
+ // sync() calls, but stubs are fast so in practice this takes
+ // much less than 1s to complete.)
+ pool.unknown = nil
+ for deadline := time.Now().Add(time.Second); ; time.Sleep(time.Millisecond) {
+ sch.sync()
+ ents, _ = queue.Entries()
+ if len(ents) == 0 || time.Now().After(deadline) {
+ break
+ }
+ }
+ c.Check(ents, check.HasLen, 0)
+}
newExecutor := func(cloud.Instance) Executor {
return &stubExecutor{
response: map[string]stubResp{
- "crunch-run --list": stubResp{},
- "true": stubResp{},
+ "crunch-run --list": {},
+ "true": {},
},
}
}
pool2.Stop()
}
+func (suite *PoolSuite) TestDrain(c *check.C) {
+ logger := ctxlog.TestLogger(c)
+ driver := test.StubDriver{}
+ instanceSet, err := driver.InstanceSet(nil, "test-instance-set-id", nil, logger)
+ c.Assert(err, check.IsNil)
+
+ ac := arvados.NewClientFromEnv()
+
+ type1 := test.InstanceType(1)
+ pool := &Pool{
+ arvClient: ac,
+ logger: logger,
+ newExecutor: func(cloud.Instance) Executor { return &stubExecutor{} },
+ instanceSet: &throttledInstanceSet{InstanceSet: instanceSet},
+ instanceTypes: arvados.InstanceTypeMap{
+ type1.Name: type1,
+ },
+ }
+ notify := pool.Subscribe()
+ defer pool.Unsubscribe(notify)
+
+ pool.Create(type1)
+
+ // Wait for the instance to either return from its Create
+ // call, or show up in a poll.
+ suite.wait(c, pool, notify, func() bool {
+ pool.mtx.RLock()
+ defer pool.mtx.RUnlock()
+ return len(pool.workers) == 1
+ })
+
+ tests := []struct {
+ state State
+ idleBehavior IdleBehavior
+ result bool
+ }{
+ {StateIdle, IdleBehaviorHold, false},
+ {StateIdle, IdleBehaviorDrain, false},
+ {StateIdle, IdleBehaviorRun, true},
+ }
+
+ for _, test := range tests {
+ for _, wkr := range pool.workers {
+ wkr.state = test.state
+ wkr.idleBehavior = test.idleBehavior
+ }
+
+ // Try to start a container
+ started := pool.StartContainer(type1, arvados.Container{UUID: "testcontainer"})
+ c.Check(started, check.Equals, test.result)
+ }
+}
+
func (suite *PoolSuite) TestCreateUnallocShutdown(c *check.C) {
logger := ctxlog.TestLogger(c)
driver := test.StubDriver{HoldCloudOps: true}
} else if arg == "debug" {
logger.SetLevel(logrus.DebugLevel)
} else {
- logger.Warnf("unkown option: %s\n", arg)
+ logger.Warnf("unknown option: %s\n", arg)
}
}
if hostname == "" || hostname == "-" {
c.Assert(err, check.IsNil)
t := time.Now().Add(-time.Hour * 24 * 365)
err = os.Chtimes(trashfile, t, t)
+ c.Assert(err, check.IsNil)
}
var stdout, stderr bytes.Buffer
continue
}
body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
c.Logf("status %d, body %s", resp.StatusCode, string(body))
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
break
def add_arv_hints():
cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")
cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
- res10 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
- res11 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
- customschema10 = res10.read().decode('utf-8')
- customschema11 = res11.read().decode('utf-8')
- use_custom_schema("v1.0", "http://arvados.org/cwl", customschema10)
- use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema11)
- use_custom_schema("v1.1", "http://arvados.org/cwl", customschema11)
- res10.close()
- res11.close()
+ supported_versions = ["v1.0", "v1.1", "v1.2"]
+ for s in supported_versions:
+ res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-%s.yml' % s)
+ customschema = res.read().decode('utf-8')
+ use_custom_schema(s, "http://arvados.org/cwl", customschema)
+ res.close()
cwltool.process.supportedProcessRequirements.extend([
"http://arvados.org/cwl#RunInSingleContainer",
"http://arvados.org/cwl#OutputDirType",
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+$base: "http://arvados.org/cwl#"
+$namespaces:
+ cwl: "https://w3id.org/cwl/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+$graph:
+- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
+
+- name: cwltool:Secrets
+ type: record
+ inVocab: false
+ extends: cwl:ProcessRequirement
+ fields:
+ class:
+ type: string
+ doc: "Always 'Secrets'"
+ jsonldPredicate:
+ "_id": "@type"
+ "_type": "@vocab"
+ secrets:
+ type: string[]
+ doc: |
+ List one or more input parameters that are sensitive (such as passwords)
+ which will be deliberately obscured from logging.
+ jsonldPredicate:
+ "_type": "@id"
+ refScope: 0
+
+- name: RunInSingleContainer
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Indicates that a subworkflow should run in a single container
+ and not be scheduled as separate steps.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:RunInSingleContainer'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+
+- name: OutputDirType
+ type: enum
+ symbols:
+ - local_output_dir
+ - keep_output_dir
+ doc:
+ - |
+ local_output_dir: Use regular file system local to the compute node.
+ There must be sufficient local scratch space to store entire output;
+ specify this with `outdirMin` of `ResourceRequirement`. Files are
+ batch uploaded to Keep when the process completes. Most compatible, but
+ upload step can be time consuming for very large files.
+ - |
+ keep_output_dir: Use writable Keep mount. Files are streamed to Keep as
+ they are written. Does not consume local scratch space, but does consume
+ RAM for output buffers (up to 192 MiB per file simultaneously open for
+ writing.) Best suited to processes which produce sequential output of
+ large files (non-sequential writes may produced fragmented file
+ manifests). Supports regular files and directories, does not support
+ special files such as symlinks, hard links, named pipes, named sockets,
+ or device nodes.
+
+
+- name: RuntimeConstraints
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Set Arvados-specific runtime hints.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:RuntimeConstraints'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ - name: keep_cache
+ type: int?
+ doc: |
+ Size of file data buffer for Keep mount in MiB. Default is 256
+ MiB. Increase this to reduce cache thrashing in situations such as
+ accessing multiple large (64+ MiB) files at the same time, or
+ performing random access on a large file.
+ - name: outputDirType
+ type: OutputDirType?
+ doc: |
+ Preferred backing store for output staging. If not specified, the
+ system may choose which one to use.
+
+- name: PartitionRequirement
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Select preferred compute partitions on which to run jobs.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:PartitionRequirement'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ - name: partition
+ type:
+ - string
+ - string[]
+
+- name: APIRequirement
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Indicates that process wants to access to the Arvados API. Will be granted
+ limited network access and have ARVADOS_API_HOST and ARVADOS_API_TOKEN set
+ in the environment.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:APIRequirement'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+
+- name: IntermediateOutput
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify desired handling of intermediate output collections.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:IntermediateOutput'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ outputTTL:
+ type: int
+ doc: |
+ If the value is greater than zero, consider intermediate output
+ collections to be temporary and should be automatically
+ trashed. Temporary collections will be trashed `outputTTL` seconds
+ after creation. A value of zero means intermediate output should be
+ retained indefinitely (this is the default behavior).
+
+ Note: arvados-cwl-runner currently does not take workflow dependencies
+ into account when setting the TTL on an intermediate output
+ collection. If the TTL is too short, it is possible for a collection to
+ be trashed before downstream steps that consume it are started. The
+ recommended minimum value for TTL is the expected duration of the
+ entire the workflow.
+
+- name: WorkflowRunnerResources
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify memory or cores resource request for the CWL runner process itself.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:WorkflowRunnerResources'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ ramMin:
+ type: int?
+ doc: Minimum RAM, in mebibytes (2**20)
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/ramMin"
+ coresMin:
+ type: int?
+ doc: Minimum cores allocated to cwl-runner
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
+ keep_cache:
+ type: int?
+ doc: |
+ Size of collection metadata cache for the workflow runner, in
+ MiB. Default 256 MiB. Will be added on to the RAM request
+ when determining node size to request.
+ jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+
+- name: ClusterTarget
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify where a workflow step should run
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:ClusterTarget'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ cluster_id:
+ type: string?
+ doc: The cluster to run the container
+ project_uuid:
+ type: string?
+ doc: The project that will own the container requests and intermediate collections
download_url="https://github.com/arvados/arvados.git",
license='Apache 2.0',
packages=find_packages(),
- package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml']},
+ package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml', 'arv-cwl-schema-v1.2.yml']},
scripts=[
'bin/cwl-runner',
'bin/arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200720165847',
+ 'cwltool==3.0.20200807132242',
'schema-salad==7.0.20200612160654',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
IncludeTrash bool `json:"include_trash"`
IncludeOldVersions bool `json:"include_old_versions"`
BypassFederation bool `json:"bypass_federation"`
+ ForwardedFor string `json:"forwarded_for,omitempty"`
}
type CreateOptions struct {
c.Check(err, check.IsNil)
c.Check(d.D, check.Equals, Duration(time.Second+234*time.Millisecond))
buf, err := json.Marshal(d)
+ c.Check(err, check.IsNil)
c.Check(string(buf), check.Equals, `{"D":"1.234s"}`)
for _, trial := range []struct {
f, err := s.fs.Open(path)
c.Assert(err, check.IsNil)
fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
c.Check(len(fis), check.Not(check.Equals), 0)
ok := false
f, err = s.fs.Open("/by_id/" + path)
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
+ c.Assert(err, check.IsNil)
var names []string
for _, fi := range fis {
names = append(names, fi.Name())
f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
+ c.Assert(err, check.IsNil)
var names []string
for _, fi := range fis {
names = append(names, fi.Name())
func (s *ServerRequiredSuite) TestGetInvalidUUID(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("collections", "", nil, &getback)
func (s *ServerRequiredSuite) TestGetValidUUID(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("collections", "zzzzz-4zz18-abcdeabcdeabcde", nil, &getback)
func (s *ServerRequiredSuite) TestInvalidResourceType(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultCollectionReplication(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
value, err := arv.Discovery("defaultCollectionReplication")
c.Assert(err, IsNil)
c.Assert(value, NotNil)
func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
value, err := arv.Discovery("noSuchParameter")
c.Assert(err, NotNil)
c.Assert(value, IsNil)
gotReq := make(map[string]interface{})
err = dec.Decode(&gotReq)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotReq)
c.Check(gotReq["RequestID"], check.Matches, "req-[a-z0-9]{20}")
c.Check(gotReq["reqForwardedFor"], check.Equals, "1.2.3.4:12345")
gotResp := make(map[string]interface{})
err = dec.Decode(&gotResp)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotResp)
c.Check(gotResp["RequestID"], check.Equals, gotReq["RequestID"])
c.Check(gotResp["reqForwardedFor"], check.Equals, "1.2.3.4:12345")
gotReq := make(map[string]interface{})
err = dec.Decode(&gotReq)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotReq)
gotResp := make(map[string]interface{})
err = dec.Decode(&gotResp)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotResp)
if trial.expectLog {
c.Check(gotResp["respBody"], check.Equals, trial.expectBody, comment)
c.Check(err, check.Equals, want)
case string:
buf := make([]byte, len(want))
- n, err := io.ReadFull(rdr, buf)
+ _, err := io.ReadFull(rdr, buf)
c.Check(err, check.IsNil)
for i := 0; i < 4; i++ {
c.Check(string(buf), check.Equals, want)
- n, err = rdr.Read(buf)
+ n, err := rdr.Read(buf)
c.Check(n, check.Equals, 0)
c.Check(err, check.Equals, io.EOF)
}
filesize := 0
for i := range locs {
_, err := rand.Read(buf[:i])
+ c.Assert(err, check.IsNil)
h.Write(buf[:i])
locs[i], _, err = s.kc.PutB(buf[:i])
c.Assert(err, check.IsNil)
offset := rand.Intn(len(buf) - 1)
count := rand.Intn(len(buf) - offset)
if rand.Intn(2) == 0 {
- curPos, err = rdr.Seek(int64(offset)-curPos, io.SeekCurrent)
+ curPos, _ = rdr.Seek(int64(offset)-curPos, io.SeekCurrent)
} else {
- curPos, err = rdr.Seek(int64(offset), io.SeekStart)
+ curPos, _ = rdr.Seek(int64(offset), io.SeekStart)
}
c.Check(curPos, check.Equals, int64(offset))
for count > 0 {
count -= n
}
curPos, err = rdr.Seek(0, io.SeekCurrent)
+ c.Check(err, check.IsNil)
c.Check(curPos, check.Equals, int64(offset))
}
c.Check(md5.Sum(buf), check.DeepEquals, md5.Sum(testdata))
c.Assert(err, Equals, nil)
kc, err := MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 2)
arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 3)
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
make(chan string, 4)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ksGateway.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, n, _, err := kc.Get(barhash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, nil)
<-st.handled
r, n, _, err = kc.Get(foohash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, BadChecksum)
content}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 3
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
# SPDX-License-Identifier: Apache-2.0
import arvados
+import arvados.errors
import json
import sys
users = apiC.users().list().execute()
check_A(users)
+
+####
+# bug 16683 tests
+
+# Check that this query returns empty, instead of returning a 500 or
+# 502 error.
+# Yes, we're asking for a group from the users endpoint. This is not a
+# mistake, this is something workbench does to populate the sharing
+# dialog.
+clusterID_B = apiB.configs().get().execute()["ClusterID"]
+i = apiB.users().list(filters=[["uuid", "in", ["%s-j7d0g-fffffffffffffff" % clusterID_B]]], count="none").execute()
+assert len(i["items"]) == 0
+
+# Check that we can create a project and give a remote user access to it
+
+tok3 = apiA.api_client_authorizations().create(body={"api_client_authorization": {"owner_uuid": by_username["case3"]}}).execute()
+tok4 = apiA.api_client_authorizations().create(body={"api_client_authorization": {"owner_uuid": by_username["case4"]}}).execute()
+
+v2_token3 = "v2/%s/%s" % (tok3["uuid"], tok3["api_token"])
+v2_token4 = "v2/%s/%s" % (tok4["uuid"], tok4["api_token"])
+
+apiB_3 = arvados.api(host=j["arvados_api_hosts"][1], token=v2_token3, insecure=True)
+apiB_4 = arvados.api(host=j["arvados_api_hosts"][1], token=v2_token4, insecure=True)
+
+assert apiB_3.users().current().execute()["uuid"] == by_username["case3"]
+assert apiB_4.users().current().execute()["uuid"] == by_username["case4"]
+
+newproject = apiB_3.groups().create(body={"group_class": "project",
+ "name":"fed test project"},
+ ensure_unique_name=True).execute()
+
+try:
+ # Expect to fail
+ apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+except arvados.errors.ApiError as e:
+ if e.resp['status'] == '404':
+ pass
+ else:
+ raise
+
+l = apiB_3.links().create(body={"link_class": "permission",
+ "name":"can_read",
+ "tail_uuid": by_username["case4"],
+ "head_uuid": newproject["uuid"]}).execute()
+
+# Expect to succeed
+apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+
+# remove permission
+apiB_3.links().delete(uuid=l["uuid"]).execute()
+
+try:
+ # Expect to fail again
+ apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+except arvados.errors.ApiError as e:
+ if e.resp['status'] == '404':
+ pass
+ else:
+ raise
+
print("Passed checks")
%r/[a-z0-9]{5}-#{uuid_prefix}-[a-z0-9]{15}/
end
+ def check_readable_uuid attr, attr_value
+ return if attr_value.nil?
+ if (r = ArvadosModel::resource_class_for_uuid attr_value)
+ unless skip_uuid_read_permission_check.include? attr
+ r = r.readable_by(current_user)
+ end
+ if r.where(uuid: attr_value).count == 0
+ errors.add(attr, "'#{attr_value}' not found")
+ end
+ else
+ # Not a valid uuid or PDH, but that (currently) is not an error.
+ end
+ end
+
def ensure_valid_uuids
specials = [system_user_uuid]
next if skip_uuid_existence_check.include? attr
attr_value = send attr
next if specials.include? attr_value
- if attr_value
- if (r = ArvadosModel::resource_class_for_uuid attr_value)
- unless skip_uuid_read_permission_check.include? attr
- r = r.readable_by(current_user)
- end
- if r.where(uuid: attr_value).count == 0
- errors.add(attr, "'#{attr_value}' not found")
- end
- end
- end
+ check_readable_uuid attr, attr_value
end
end
end
protected
+ def check_readable_uuid attr, attr_value
+ if attr == 'tail_uuid' &&
+ !attr_value.nil? &&
+ self.link_class == 'permission' &&
+ attr_value[0..4] != Rails.configuration.ClusterID &&
+ ApiClientAuthorization.remote_host(uuid_prefix: attr_value[0..4]) &&
+ ArvadosModel::resource_class_for_uuid(attr_value) == User
+ # Permission link tail is a remote user (the user permissions
+ # are being granted to), so bypass the standard check that a
+ # referenced object uuid is readable by current user.
+ #
+ # We could do a call to the remote cluster to check if the user
+ # in tail_uuid exists. This would detect copy-and-paste errors,
+ # but add another way for the request to fail, and I don't think
+ # it would improve security. It doesn't seem to be worth the
+ # complexity tradeoff.
+ true
+ else
+ super
+ end
+ end
+
def permission_to_attach_to_objects
# Anonymous users cannot write links
return false if !current_user
head_obj = ArvadosModel.find_by_uuid(head_uuid)
+ if head_obj.nil?
+ errors.add(:head_uuid, "does not exist")
+ return false
+ end
+
# No permission links can be pointed to past collection versions
if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
errors.add(:head_uuid, "cannot point to a past version of a collection")
users(:active).uuid.sub(/-\w+$/, "-#{'z' * 15}"))
end
+ test "link granting permission to remote user is valid" do
+ refute new_active_link_valid?(tail_uuid:
+ users(:active).uuid.sub(/^\w+-/, "foooo-"))
+ Rails.configuration.RemoteClusters = Rails.configuration.RemoteClusters.merge({foooo: ActiveSupport::InheritableOptions.new({Host: "bar.com"})})
+ assert new_active_link_valid?(tail_uuid:
+ users(:active).uuid.sub(/^\w+-/, "foooo-"))
+ end
+
test "link granting non-project permission to unreadable user is invalid" do
refute new_active_link_valid?(tail_uuid: users(:admin).uuid,
head_uuid: collections(:bar_file).uuid)
c.Fatal(err)
}
_, p, err := net.SplitHostPort(ln.Addr().String())
+ c.Check(err, check.IsNil)
ln.Close()
config := "Clusters:\n zzzzz:\n ManagementToken: abcdefg\n Services: {Keepbalance: {InternalURLs: {'http://localhost:" + p + "/': {}}}}\n"
+++ /dev/null
-#! /usr/bin/env python
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# traffic_test.py
-#
-# Launch a test Keep and API server and PUT and GET a bunch of blocks.
-# Can be used to simulate client traffic in Keep to evaluate memory usage,
-# error logging, performance, etc.
-#
-# This script is warty and is relatively environment-specific, but the
-# example run described below should execute cleanly.
-#
-# Usage:
-# traffic_test.py start
-# Starts the test servers.
-# traffic_test.py put file1 file2 file3 ....
-# Runs arv-put on each file.
-# traffic_test.py get hash1 hash2 hash3 ....
-# Loops forever issuing GET requests for specified blocks.
-# traffic_test.py stop
-# Stops the test servers.
-#
-# Example:
-#
-# $ ./traffic_test.py start
-# $ ./traffic_test.py put GS00253-DNA_A02_200_37.tsv.bz2 \
-# GS00253-DNA_B01_200_37.tsv.bz2 \
-# GS00253-DNA_B02_200_37.tsv.bz2
-# $ ./traffic_test.py get $(find /tmp/tmp* -type f -printf "%f ")
-# [loops forever]
-# ^C
-# $ ./traffic_test.py stop
-#
-# Multiple "get" runs may be run concurrently to evaluate Keep's handling
-# of additional concurrent clients.
-
-PYSDK_DIR = "../../../sdk/python"
-PYTEST_DIR = PYSDK_DIR + "/tests"
-ARV_PUT_PATH = PYSDK_DIR + "/bin/arv-put"
-ARV_GET_PATH = PYSDK_DIR + "/bin/arv-get"
-SECONDS_BETWEEN_GETS = 1
-
-import argparse
-import httplib2
-import os
-import random
-import subprocess
-import sys
-import time
-
-# for run_test_server.py
-sys.path.insert(0, PYSDK_DIR)
-sys.path.insert(0, PYTEST_DIR)
-import arvados
-import run_test_server
-
-def arv_cmd(*args):
- p = subprocess.Popen([sys.executable] + list(args),
- stdout=subprocess.PIPE)
- (arvout, arverr) = p.communicate()
- if p.returncode != 0:
- print "error {} from {} {}: {}".format(
- p.returncode, sys.executable, args, arverr)
- sys.exit(p.returncode)
- return arvout
-
-def start():
- run_test_server.run()
- run_test_server.run_keep()
-
-def put(files):
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
- run_test_server.authorize_with('active')
- for v in ["ARVADOS_API_HOST",
- "ARVADOS_API_HOST_INSECURE",
- "ARVADOS_API_TOKEN"]:
- os.environ[v] = arvados.config.settings()[v]
-
- if not os.environ.has_key('PYTHONPATH'):
- os.environ['PYTHONPATH'] = ''
- os.environ['PYTHONPATH'] = "{}:{}:{}".format(
- PYSDK_DIR, PYTEST_DIR, os.environ['PYTHONPATH'])
-
- for c in files:
- manifest_uuid = arv_cmd(ARV_PUT_PATH, c)
-
-def get(blocks):
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
-
- run_test_server.authorize_with('active')
- for v in ["ARVADOS_API_HOST",
- "ARVADOS_API_HOST_INSECURE",
- "ARVADOS_API_TOKEN"]:
- os.environ[v] = arvados.config.settings()[v]
-
- nqueries = 0
- while True:
- b = random.choice(blocks)
- print "GET /" + b
- body = arv_cmd(ARV_GET_PATH, b)
- print "got {} bytes".format(len(body))
- time.sleep(SECONDS_BETWEEN_GETS)
- nqueries = nqueries + 1
-
-def stop():
- run_test_server.stop_keep()
- run_test_server.stop()
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument('action',
- type=str,
- nargs='+',
- help='''"start", "put", "get", "stop"''')
- args = parser.parse_args()
-
- if args.action[0] == 'start':
- start()
- elif args.action[0] == 'put':
- put(args.action[1:])
- elif args.action[0] == 'get':
- get(args.action[1:])
- elif args.action[0] == 'stop':
- stop()
- else:
- print('Unrecognized action "{}"'.format(args.action))
- print('actions are "start", "put", "get", "stop"')
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err := v.check(s.metadata.URL + "/latest")
+ c.Check(err, check.IsNil)
creds, err := v.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
+ c.Check(err, check.IsNil)
c.Check(creds.AccessKeyID, check.Equals, "ASIAIOSFODNN7EXAMPLE")
c.Check(creds.SecretAccessKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err = deadv.check(s.metadata.URL + "/latest")
+ c.Check(err, check.IsNil)
_, err = deadv.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
c.Check(err, check.ErrorMatches, `(?s).*EC2RoleRequestError: no EC2 instance role found.*`)
c.Check(err, check.ErrorMatches, `(?s).*404.*`)
projects / arvados.git / commitdiff