[User, Group].each do |type|
type
.filter([['uuid','in',@share_links.collect(&:tail_uuid)]])
+ .with_count("none")
+ .fetch_multiple_pages(false)
.each do |o|
uuid_map[o.uuid] = o
end
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-What you need to know and do in order to upgrade your Arvados installation.
+For Arvados administrators, this page will cover what you need to know and do in order to ensure a smooth upgrade of your Arvados installation. For general release notes covering features added and bugs fixed, see "Arvados releases":https://arvados.org/releases .
h2. General process
# Consult upgrade notes below to see if any manual configuration updates are necessary.
# Wait for the cluster to be idle and stop Arvados services.
# Install new packages using @apt-get upgrade@ or @yum upgrade@.
-# Package installation scripts will perform any necessary data migrations.
+# Wait for package installation scripts as they perform any necessary data migrations.
# Restart Arvados services.
h2. Upgrade notes
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.ContainerList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.SpecimenList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.UserList(ctx, options)
if err != nil {
return nil, err
var needSort atomic.Value
needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ options.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor
cl, err := backend.CollectionList(ctx, options)
if err != nil {
return nil, err
// backend.
func (conn *Conn) splitListRequest(ctx context.Context, opts arvados.ListOptions, fn func(context.Context, string, arvados.API, arvados.ListOptions) ([]string, error)) error {
- if opts.BypassFederation {
+ if opts.BypassFederation || opts.ForwardedFor != "" {
// Client requested no federation. Pass through.
_, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
return err
done, err := fn(ctx, clusterID, backend, remoteOpts)
if err != nil {
- errs <- httpErrorf(http.StatusBadGateway, err.Error())
+ errs <- httpErrorf(http.StatusBadGateway, "%s", err.Error())
return
}
progress := false
db.SetMaxOpenConns(p)
}
if err := db.Ping(); err != nil {
- ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect scuceeded but ping failed")
+ ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect succeeded but ping failed")
return nil, errDBConnection
}
h.pgdb = db
resp, err = arvados.InsecureHTTPClient.Do(req)
if c.Check(err, check.IsNil) {
err = json.NewDecoder(resp.Body).Decode(&cr)
+ c.Check(err, check.IsNil)
c.Check(cr.UUID, check.Matches, "z2222-.*")
}
}
c.Check(rr.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.HasLen, 27)
c.Check(jresp["state"], check.Equals, "Locked")
- _, rr, jresp = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
+ _, rr, _ = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
c.Check(rr.Code, check.Equals, http.StatusUnprocessableEntity)
c.Check(rr.Body.String(), check.Not(check.Matches), `.*"uuid":.*`)
_, rr, jresp = doRequest(c, s.rtr, token, "POST", "/arvados/v1/containers/"+uuid+"/unlock", nil, nil)
c.Check(sp.Properties["foo"], check.Equals, "bar")
spGet, err := s.conn.SpecimenGet(s.ctx, arvados.GetOptions{UUID: sp.UUID})
+ c.Check(err, check.IsNil)
c.Check(spGet.UUID, check.Equals, sp.UUID)
c.Check(spGet.Properties["foo"], check.Equals, "bar")
spList, err := s.conn.SpecimenList(s.ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(err, check.IsNil)
c.Check(spList.ItemsAvailable, check.Equals, 1)
c.Assert(spList.Items, check.HasLen, 1)
c.Check(spList.Items[0].UUID, check.Equals, sp.UUID)
anonCtx := context.WithValue(context.Background(), contextKeyTestTokens, []string{arvadostest.AnonymousToken})
spList, err = s.conn.SpecimenList(anonCtx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", sp.UUID}}})
+ c.Check(err, check.IsNil)
c.Check(spList.ItemsAvailable, check.Equals, 0)
c.Check(spList.Items, check.HasLen, 0)
spDel, err := s.conn.SpecimenDelete(s.ctx, arvados.DeleteOptions{UUID: sp.UUID})
+ c.Check(err, check.IsNil)
c.Check(spDel.UUID, check.Equals, sp.UUID)
}
notify <-chan struct{}
unalloc map[arvados.InstanceType]int // idle+booting+unknown
idle map[arvados.InstanceType]int
+ unknown map[arvados.InstanceType]int
running map[string]time.Time
atQuota bool
canCreate int
defer p.Unlock()
r := map[arvados.InstanceType]int{}
for it, n := range p.unalloc {
- r[it] = n
+ r[it] = n - p.unknown[it]
}
return r
}
worker.StateBooting: len(p.unalloc) - len(p.idle),
worker.StateIdle: len(p.idle),
worker.StateRunning: len(p.running),
+ worker.StateUnknown: len(p.unknown),
}
}
func (p *stubPool) StartContainer(it arvados.InstanceType, ctr arvados.Container) bool {
"fmt"
"git.arvados.org/arvados.git/lib/dispatchcloud/container"
+ "git.arvados.org/arvados.git/lib/dispatchcloud/worker"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/sirupsen/logrus"
)
// Running containers whose crunch-run processes have exited are
// cancelled.
func (sch *Scheduler) sync() {
+ anyUnknownWorkers := sch.pool.CountWorkers()[worker.StateUnknown] > 0
running := sch.pool.Running()
qEntries, qUpdated := sch.queue.Entries()
for uuid, ent := range qEntries {
switch ent.Container.State {
case arvados.ContainerStateRunning:
if !running {
- go sch.cancel(uuid, "not running on any worker")
+ if !anyUnknownWorkers {
+ go sch.cancel(uuid, "not running on any worker")
+ }
} else if !exited.IsZero() && qUpdated.After(exited) {
go sch.cancel(uuid, "state=Running after crunch-run exited")
} else if ent.Container.Priority == 0 {
ents, _ = queue.Entries()
c.Check(ents, check.HasLen, 0)
}
+
+func (*SchedulerSuite) TestCancelOrphanedContainers(c *check.C) {
+ ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c))
+ pool := stubPool{
+ unalloc: map[arvados.InstanceType]int{test.InstanceType(1): 1},
+ unknown: map[arvados.InstanceType]int{test.InstanceType(1): 1},
+ }
+ queue := test.Queue{
+ ChooseType: chooseType,
+ Containers: []arvados.Container{
+ {
+ UUID: test.ContainerUUID(1),
+ Priority: 0,
+ State: arvados.ContainerStateRunning,
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 1,
+ RAM: 1 << 30,
+ },
+ },
+ },
+ }
+ queue.Update()
+
+ ents, _ := queue.Entries()
+ c.Check(ents, check.HasLen, 1)
+
+ sch := New(ctx, &queue, &pool, time.Millisecond, time.Millisecond)
+
+ // Sync shouldn't cancel the container because it might be
+ // running on the VM with state=="unknown".
+ //
+ // (Cancel+forget happens asynchronously and requires multiple
+ // sync() calls, so even after 10x sync-and-sleep iterations,
+ // we aren't 100% confident that sync isn't trying to
+ // cancel. But in the test environment, the goroutines started
+ // by sync() access stubs and therefore run quickly, so it
+ // works fine in practice. We accept that if the code is
+ // broken, the test will still pass occasionally.)
+ for i := 0; i < 10; i++ {
+ sch.sync()
+ time.Sleep(time.Millisecond)
+ }
+ ents, _ = queue.Entries()
+ c.Check(ents, check.HasLen, 1)
+ c.Check(ents[test.ContainerUUID(1)].Container.State, check.Equals, arvados.ContainerStateRunning)
+
+ // Sync should cancel & forget the container when the
+ // "unknown" node goes away.
+ //
+ // (As above, cancel+forget is async and requires multiple
+ // sync() calls, but stubs are fast so in practice this takes
+ // much less than 1s to complete.)
+ pool.unknown = nil
+ for deadline := time.Now().Add(time.Second); ; time.Sleep(time.Millisecond) {
+ sch.sync()
+ ents, _ = queue.Entries()
+ if len(ents) == 0 || time.Now().After(deadline) {
+ break
+ }
+ }
+ c.Check(ents, check.HasLen, 0)
+}
newExecutor := func(cloud.Instance) Executor {
return &stubExecutor{
response: map[string]stubResp{
- "crunch-run --list": stubResp{},
- "true": stubResp{},
+ "crunch-run --list": {},
+ "true": {},
},
}
}
pool2.Stop()
}
+func (suite *PoolSuite) TestDrain(c *check.C) {
+ logger := ctxlog.TestLogger(c)
+ driver := test.StubDriver{}
+ instanceSet, err := driver.InstanceSet(nil, "test-instance-set-id", nil, logger)
+ c.Assert(err, check.IsNil)
+
+ ac := arvados.NewClientFromEnv()
+
+ type1 := test.InstanceType(1)
+ pool := &Pool{
+ arvClient: ac,
+ logger: logger,
+ newExecutor: func(cloud.Instance) Executor { return &stubExecutor{} },
+ instanceSet: &throttledInstanceSet{InstanceSet: instanceSet},
+ instanceTypes: arvados.InstanceTypeMap{
+ type1.Name: type1,
+ },
+ }
+ notify := pool.Subscribe()
+ defer pool.Unsubscribe(notify)
+
+ pool.Create(type1)
+
+ // Wait for the instance to either return from its Create
+ // call, or show up in a poll.
+ suite.wait(c, pool, notify, func() bool {
+ pool.mtx.RLock()
+ defer pool.mtx.RUnlock()
+ return len(pool.workers) == 1
+ })
+
+ tests := []struct {
+ state State
+ idleBehavior IdleBehavior
+ result bool
+ }{
+ {StateIdle, IdleBehaviorHold, false},
+ {StateIdle, IdleBehaviorDrain, false},
+ {StateIdle, IdleBehaviorRun, true},
+ }
+
+ for _, test := range tests {
+ for _, wkr := range pool.workers {
+ wkr.state = test.state
+ wkr.idleBehavior = test.idleBehavior
+ }
+
+ // Try to start a container
+ started := pool.StartContainer(type1, arvados.Container{UUID: "testcontainer"})
+ c.Check(started, check.Equals, test.result)
+ }
+}
+
func (suite *PoolSuite) TestCreateUnallocShutdown(c *check.C) {
logger := ctxlog.TestLogger(c)
driver := test.StubDriver{HoldCloudOps: true}
continue
}
body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
c.Logf("status %d, body %s", resp.StatusCode, string(body))
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
break
s.summary = "Arvados CLI tools"
s.description = "Arvados command line tools, git commit #{git_hash}"
s.authors = ["Arvados Authors"]
- s.email = 'gem-dev@curoverse.com'
+ s.email = 'gem-dev@arvados.org'
#s.bindir = '.'
s.licenses = ['Apache-2.0']
s.files = ["bin/arv", "bin/arv-tag", "LICENSE-2.0.txt"]
# Our google-api-client dependency used to be < 0.9, but that could be
# satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
s.add_runtime_dependency 'arvados-google-api-client', '~> 0.6', '>= 0.6.3', '<0.8.9'
- s.add_runtime_dependency 'activesupport', '>= 3.2.13', '< 5.1'
+ s.add_runtime_dependency 'activesupport', '>= 3.2.13', '< 5.3'
s.add_runtime_dependency 'json', '>= 1.7.7', '<3'
s.add_runtime_dependency 'optimist', '~> 3.0'
s.add_runtime_dependency 'andand', '~> 1.3', '>= 1.3.3'
IncludeTrash bool `json:"include_trash"`
IncludeOldVersions bool `json:"include_old_versions"`
BypassFederation bool `json:"bypass_federation"`
+ ForwardedFor string `json:"forwarded_for,omitempty"`
}
type CreateOptions struct {
c.Check(err, check.IsNil)
c.Check(d.D, check.Equals, Duration(time.Second+234*time.Millisecond))
buf, err := json.Marshal(d)
+ c.Check(err, check.IsNil)
c.Check(string(buf), check.Equals, `{"D":"1.234s"}`)
for _, trial := range []struct {
f, err := s.fs.Open(path)
c.Assert(err, check.IsNil)
fis, err := f.Readdir(-1)
+ c.Assert(err, check.IsNil)
c.Check(len(fis), check.Not(check.Equals), 0)
ok := false
f, err = s.fs.Open("/by_id/" + path)
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
+ c.Assert(err, check.IsNil)
var names []string
for _, fi := range fis {
names = append(names, fi.Name())
f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
c.Assert(err, check.IsNil)
fis, err = f.Readdir(-1)
+ c.Assert(err, check.IsNil)
var names []string
for _, fi := range fis {
names = append(names, fi.Name())
func (s *ServerRequiredSuite) TestGetInvalidUUID(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("collections", "", nil, &getback)
func (s *ServerRequiredSuite) TestGetValidUUID(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("collections", "zzzzz-4zz18-abcdeabcdeabcde", nil, &getback)
func (s *ServerRequiredSuite) TestInvalidResourceType(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
getback := make(Dict)
err = arv.Get("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultCollectionReplication(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
value, err := arv.Discovery("defaultCollectionReplication")
c.Assert(err, IsNil)
c.Assert(value, NotNil)
func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
arv, err := MakeArvadosClient()
+ c.Assert(err, IsNil)
value, err := arv.Discovery("noSuchParameter")
c.Assert(err, NotNil)
c.Assert(value, IsNil)
gotReq := make(map[string]interface{})
err = dec.Decode(&gotReq)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotReq)
c.Check(gotReq["RequestID"], check.Matches, "req-[a-z0-9]{20}")
c.Check(gotReq["reqForwardedFor"], check.Equals, "1.2.3.4:12345")
gotResp := make(map[string]interface{})
err = dec.Decode(&gotResp)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotResp)
c.Check(gotResp["RequestID"], check.Equals, gotReq["RequestID"])
c.Check(gotResp["reqForwardedFor"], check.Equals, "1.2.3.4:12345")
gotReq := make(map[string]interface{})
err = dec.Decode(&gotReq)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotReq)
gotResp := make(map[string]interface{})
err = dec.Decode(&gotResp)
+ c.Check(err, check.IsNil)
c.Logf("%#v", gotResp)
if trial.expectLog {
c.Check(gotResp["respBody"], check.Equals, trial.expectBody, comment)
c.Assert(err, Equals, nil)
kc, err := MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 2)
arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 3)
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
make(chan string, 4)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ksGateway.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, n, _, err := kc.Get(barhash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, nil)
<-st.handled
r, n, _, err = kc.Get(foohash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, BadChecksum)
content}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 3
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
# SPDX-License-Identifier: Apache-2.0
import arvados
+import arvados.errors
import json
import sys
users = apiC.users().list().execute()
check_A(users)
+
+####
+# bug 16683 tests
+
+# Check that this query returns empty, instead of returning a 500 or
+# 502 error.
+# Yes, we're asking for a group from the users endpoint. This is not a
+# mistake, this is something workbench does to populate the sharing
+# dialog.
+clusterID_B = apiB.configs().get().execute()["ClusterID"]
+i = apiB.users().list(filters=[["uuid", "in", ["%s-j7d0g-fffffffffffffff" % clusterID_B]]], count="none").execute()
+assert len(i["items"]) == 0
+
+# Check that we can create a project and give a remote user access to it
+
+tok3 = apiA.api_client_authorizations().create(body={"api_client_authorization": {"owner_uuid": by_username["case3"]}}).execute()
+tok4 = apiA.api_client_authorizations().create(body={"api_client_authorization": {"owner_uuid": by_username["case4"]}}).execute()
+
+v2_token3 = "v2/%s/%s" % (tok3["uuid"], tok3["api_token"])
+v2_token4 = "v2/%s/%s" % (tok4["uuid"], tok4["api_token"])
+
+apiB_3 = arvados.api(host=j["arvados_api_hosts"][1], token=v2_token3, insecure=True)
+apiB_4 = arvados.api(host=j["arvados_api_hosts"][1], token=v2_token4, insecure=True)
+
+assert apiB_3.users().current().execute()["uuid"] == by_username["case3"]
+assert apiB_4.users().current().execute()["uuid"] == by_username["case4"]
+
+newproject = apiB_3.groups().create(body={"group_class": "project",
+ "name":"fed test project"},
+ ensure_unique_name=True).execute()
+
+try:
+ # Expect to fail
+ apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+except arvados.errors.ApiError as e:
+ if e.resp['status'] == '404':
+ pass
+ else:
+ raise
+
+l = apiB_3.links().create(body={"link_class": "permission",
+ "name":"can_read",
+ "tail_uuid": by_username["case4"],
+ "head_uuid": newproject["uuid"]}).execute()
+
+# Expect to succeed
+apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+
+# remove permission
+apiB_3.links().delete(uuid=l["uuid"]).execute()
+
+try:
+ # Expect to fail again
+ apiB_4.groups().get(uuid=newproject["uuid"]).execute()
+except arvados.errors.ApiError as e:
+ if e.resp['status'] == '404':
+ pass
+ else:
+ raise
+
print("Passed checks")
source 'https://rubygems.org'
-gem 'rails', '~> 5.0.0'
+gem 'rails', '~> 5.2.0'
gem 'responders', '~> 2.0'
group :test, :development do
gem 'factory_bot_rails'
- gem 'database_cleaner'
# As of now (2019-03-27) There's an open issue about incompatibilities with
# newer versions of this gem: https://github.com/rails/rails-perftest/issues/38
gem 'simplecov-rcov', require: false
gem 'mocha', require: false
gem 'byebug'
+ gem 'listen'
end
+# Fast app boot times
+gem 'bootsnap', require: false
+
gem 'pg', '~> 1.0'
gem 'multi_json'
GEM
remote: https://rubygems.org/
specs:
- actioncable (5.0.7.2)
- actionpack (= 5.0.7.2)
- nio4r (>= 1.2, < 3.0)
- websocket-driver (~> 0.6.1)
- actionmailer (5.0.7.2)
- actionpack (= 5.0.7.2)
- actionview (= 5.0.7.2)
- activejob (= 5.0.7.2)
+ actioncable (5.2.4.3)
+ actionpack (= 5.2.4.3)
+ nio4r (~> 2.0)
+ websocket-driver (>= 0.6.1)
+ actionmailer (5.2.4.3)
+ actionpack (= 5.2.4.3)
+ actionview (= 5.2.4.3)
+ activejob (= 5.2.4.3)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.0.7.2)
- actionview (= 5.0.7.2)
- activesupport (= 5.0.7.2)
- rack (~> 2.0)
- rack-test (~> 0.6.3)
+ actionpack (5.2.4.3)
+ actionview (= 5.2.4.3)
+ activesupport (= 5.2.4.3)
+ rack (~> 2.0, >= 2.0.8)
+ rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.0.7.2)
- activesupport (= 5.0.7.2)
+ actionview (5.2.4.3)
+ activesupport (= 5.2.4.3)
builder (~> 3.1)
- erubis (~> 2.7.0)
+ erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.0.7.2)
- activesupport (= 5.0.7.2)
+ activejob (5.2.4.3)
+ activesupport (= 5.2.4.3)
globalid (>= 0.3.6)
- activemodel (5.0.7.2)
- activesupport (= 5.0.7.2)
- activerecord (5.0.7.2)
- activemodel (= 5.0.7.2)
- activesupport (= 5.0.7.2)
- arel (~> 7.0)
- activesupport (5.0.7.2)
+ activemodel (5.2.4.3)
+ activesupport (= 5.2.4.3)
+ activerecord (5.2.4.3)
+ activemodel (= 5.2.4.3)
+ activesupport (= 5.2.4.3)
+ arel (>= 9.0)
+ activestorage (5.2.4.3)
+ actionpack (= 5.2.4.3)
+ activerecord (= 5.2.4.3)
+ marcel (~> 0.3.1)
+ activesupport (5.2.4.3)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
addressable (2.7.0)
public_suffix (>= 2.0.2, < 5.0)
andand (1.3.3)
- arel (7.1.4)
- arvados-google-api-client (0.8.7.3)
- activesupport (>= 3.2, < 5.1)
+ arel (9.0.0)
+ arvados-google-api-client (0.8.7.4)
+ activesupport (>= 3.2, < 5.3)
addressable (~> 2.3)
autoparse (~> 0.3)
extlib (~> 0.9)
addressable (>= 2.3.1)
extlib (>= 0.9.15)
multi_json (>= 1.0.0)
- builder (3.2.3)
+ bootsnap (1.4.7)
+ msgpack (~> 1.0)
+ builder (3.2.4)
byebug (11.0.1)
capistrano (2.15.9)
highline
net-sftp (>= 2.0.0)
net-ssh (>= 2.0.14)
net-ssh-gateway (>= 1.1.0)
- concurrent-ruby (1.1.5)
- crass (1.0.4)
- database_cleaner (1.7.0)
- erubis (2.7.0)
+ concurrent-ruby (1.1.6)
+ crass (1.0.6)
+ erubi (1.9.0)
execjs (2.7.0)
extlib (0.9.16)
factory_bot (5.0.2)
launchy (2.4.3)
addressable (~> 2.3)
libv8 (3.16.14.19)
+ listen (3.2.1)
+ rb-fsevent (~> 0.10, >= 0.10.3)
+ rb-inotify (~> 0.9, >= 0.9.10)
lograge (0.10.0)
actionpack (>= 4)
activesupport (>= 4)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.2.3)
+ loofah (2.6.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
+ marcel (0.3.3)
+ mimemagic (~> 0.3.2)
memoist (0.16.2)
metaclass (0.0.4)
- method_source (0.9.2)
- mini_mime (1.0.1)
+ method_source (1.0.0)
+ mimemagic (0.3.5)
+ mini_mime (1.0.2)
mini_portile2 (2.4.0)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
+ msgpack (1.3.3)
multi_json (1.14.1)
multi_xml (0.6.0)
multipart-post (2.1.1)
net-ssh (5.2.0)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
- nio4r (2.3.1)
- nokogiri (1.10.8)
+ nio4r (2.5.2)
+ nokogiri (1.10.10)
mini_portile2 (~> 2.4.0)
oauth2 (1.4.1)
faraday (>= 0.8, < 0.16.0)
power_assert (1.1.4)
public_suffix (4.0.3)
rack (2.2.3)
- rack-test (0.6.3)
- rack (>= 1.0)
- rails (5.0.7.2)
- actioncable (= 5.0.7.2)
- actionmailer (= 5.0.7.2)
- actionpack (= 5.0.7.2)
- actionview (= 5.0.7.2)
- activejob (= 5.0.7.2)
- activemodel (= 5.0.7.2)
- activerecord (= 5.0.7.2)
- activesupport (= 5.0.7.2)
+ rack-test (1.1.0)
+ rack (>= 1.0, < 3)
+ rails (5.2.4.3)
+ actioncable (= 5.2.4.3)
+ actionmailer (= 5.2.4.3)
+ actionpack (= 5.2.4.3)
+ actionview (= 5.2.4.3)
+ activejob (= 5.2.4.3)
+ activemodel (= 5.2.4.3)
+ activerecord (= 5.2.4.3)
+ activestorage (= 5.2.4.3)
+ activesupport (= 5.2.4.3)
bundler (>= 1.3.0)
- railties (= 5.0.7.2)
+ railties (= 5.2.4.3)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
- rails-html-sanitizer (1.0.4)
- loofah (~> 2.2, >= 2.2.2)
+ rails-html-sanitizer (1.3.0)
+ loofah (~> 2.3)
rails-observers (0.1.5)
activemodel (>= 4.0)
rails-perftest (0.0.7)
- railties (5.0.7.2)
- actionpack (= 5.0.7.2)
- activesupport (= 5.0.7.2)
+ railties (5.2.4.3)
+ actionpack (= 5.2.4.3)
+ activesupport (= 5.2.4.3)
method_source
rake (>= 0.8.7)
- thor (>= 0.18.1, < 2.0)
+ thor (>= 0.19.0, < 2.0)
rake (13.0.1)
rb-fsevent (0.10.3)
rb-inotify (0.9.10)
therubyracer (0.12.3)
libv8 (~> 3.16.14.15)
ref
- thor (0.20.3)
+ thor (1.0.1)
thread_safe (0.3.6)
tilt (2.0.8)
- tzinfo (1.2.6)
+ tzinfo (1.2.7)
thread_safe (~> 0.1)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.6.5)
+ websocket-driver (0.7.3)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
acts_as_api
andand
arvados!
+ bootsnap
byebug
- database_cleaner
factory_bot_rails
httpclient
jquery-rails
+ listen
lograge
logstash-event
minitest (= 5.10.3)
optimist
passenger
pg (~> 1.0)
- rails (~> 5.0.0)
+ rails (~> 5.2.0)
rails-controller-testing
rails-observers
rails-perftest
uglifier (~> 2.0)
BUNDLED WITH
- 1.16.6
+ 1.17.3
:with => :render_error)
rescue_from(ActiveRecord::RecordNotFound,
ActionController::RoutingError,
- ActionController::UnknownController,
AbstractController::ActionNotFound,
:with => :render_not_found)
end
%w(created_at modified_by_client_uuid modified_by_user_uuid modified_at).each do |x|
@attrs.delete x.to_sym
end
- @attrs = @attrs.symbolize_keys if @attrs.is_a? HashWithIndifferentAccess
+ @attrs = @attrs.symbolize_keys if @attrs.is_a? ActiveSupport::HashWithIndifferentAccess
@attrs
end
end
def log_update
- super unless (changed - UNLOGGED_CHANGES).empty?
+
+ super unless (saved_changes.keys - UNLOGGED_CHANGES).empty?
end
end
include DbCurrentTime
extend RecordFilters
+ after_find :schedule_restoring_changes
after_initialize :log_start_state
before_save :ensure_permission_to_save
before_save :ensure_owner_uuid_is_permitted
def reload(*args)
super
log_start_state
+ self
end
def self.create raw_params={}, *args
%r/[a-z0-9]{5}-#{uuid_prefix}-[a-z0-9]{15}/
end
+ def check_readable_uuid attr, attr_value
+ return if attr_value.nil?
+ if (r = ArvadosModel::resource_class_for_uuid attr_value)
+ unless skip_uuid_read_permission_check.include? attr
+ r = r.readable_by(current_user)
+ end
+ if r.where(uuid: attr_value).count == 0
+ errors.add(attr, "'#{attr_value}' not found")
+ end
+ else
+ # Not a valid uuid or PDH, but that (currently) is not an error.
+ end
+ end
+
def ensure_valid_uuids
specials = [system_user_uuid]
next if skip_uuid_existence_check.include? attr
attr_value = send attr
next if specials.include? attr_value
- if attr_value
- if (r = ArvadosModel::resource_class_for_uuid attr_value)
- unless skip_uuid_read_permission_check.include? attr
- r = r.readable_by(current_user)
- end
- if r.where(uuid: attr_value).count == 0
- errors.add(attr, "'#{attr_value}' not found")
- end
- end
- end
+ check_readable_uuid attr, attr_value
end
end
end
Rails.configuration.AuditLogs.MaxDeleteBatch.to_i > 0)
end
+ def schedule_restoring_changes
+ # This will be checked at log_start_state, to reset any (virtual) changes
+ # produced by the act of reading a serialized attribute.
+ @fresh_from_database = true
+ end
+
def log_start_state
if is_audit_logging_enabled?
@old_attributes = Marshal.load(Marshal.dump(attributes))
@old_logged_attributes = Marshal.load(Marshal.dump(logged_attributes))
+ if @fresh_from_database
+ # This instance was created from reading a database record. Attributes
+ # haven't been changed, but those serialized attributes will be reported
+ # as unpersisted, so we restore them to avoid issues with lock!() and
+ # with_lock().
+ restore_attributes
+ @fresh_from_database = nil
+ end
end
end
should_preserve_version = should_preserve_version? # Time sensitive, cache value
return(yield) unless (should_preserve_version || syncable_updates.any?)
- # Put aside the changes because with_lock forces a record reload
+ # Put aside the changes because with_lock does a record reload
changes = self.changes
snapshot = nil
+ restore_attributes
with_lock do
# Copy the original state to save it as old version
if should_preserve_version
def syncable_updates
updates = {}
- (syncable_attrs & self.changes.keys).each do |attr|
+ if self.changes.any?
+ changes = self.changes
+ else
+ # If called after save...
+ changes = self.saved_changes
+ end
+ (syncable_attrs & changes.keys).each do |attr|
if attr == 'uuid'
# Point old versions to current version's new UUID
- updates['current_version_uuid'] = self.changes[attr].last
+ updates['current_version_uuid'] = changes[attr].last
else
- updates[attr] = self.changes[attr].last
+ updates[attr] = changes[attr].last
end
end
return updates
def sync_past_versions
updates = self.syncable_updates
- Collection.where('current_version_uuid = ? AND uuid != ?', self.uuid_was, self.uuid_was).each do |c|
+ Collection.where('current_version_uuid = ? AND uuid != ?', self.uuid_before_last_save, self.uuid_before_last_save).each do |c|
c.attributes = updates
# Use a different validation context to skip the 'past_versions_cannot_be_updated'
# validator, as on this case it is legal to update some fields.
end
def propagate_priority
- return true unless priority_changed?
+ return true unless saved_change_to_priority?
act_as_system_user do
# Update the priority of child container requests to match new
# priority of the parent container (ignoring requests with no
if users_list.select { |u| u.is_admin }.any?
return super
end
- Container.where(ContainerRequest.readable_by(*users_list).where("containers.uuid = container_requests.container_uuid").exists)
+ Container.where(ContainerRequest.readable_by(*users_list).where("containers.uuid = container_requests.container_uuid").arel.exists)
end
def final?
# If self.final?, this update is superfluous: the final log/output
# update will be done when handle_completed calls finalize! on
# each requesting CR.
- return if self.final? || !self.log_changed?
+ return if self.final? || !saved_change_to_log?
leave_modified_by_user_alone do
ContainerRequest.where(container_uuid: self.uuid).each do |cr|
cr.update_collections(container: self, collections: ['log'])
def handle_completed
# This container is finished so finalize any associated container requests
# that are associated with this container.
- if self.state_changed? and self.final?
+ if saved_change_to_state? and self.final?
# These get wiped out by with_lock (which reloads the record),
# so record them now in case we need to schedule a retry.
- prev_secret_mounts = self.secret_mounts_was
- prev_runtime_token = self.runtime_token_was
+ prev_secret_mounts = secret_mounts_before_last_save
+ prev_runtime_token = runtime_token_before_last_save
# Need to take a lock on the container to ensure that any
# concurrent container requests that might try to reuse this
end
def update_priority
- return unless state_changed? || priority_changed? || container_uuid_changed?
+ return unless saved_change_to_state? || saved_change_to_priority? || saved_change_to_container_uuid?
act_as_system_user do
Container.
- where('uuid in (?)', [self.container_uuid_was, self.container_uuid].compact).
+ where('uuid in (?)', [container_uuid_before_last_save, self.container_uuid].compact).
map(&:update_priority!)
end
end
end
def update_trash
- if trash_at_changed? or owner_uuid_changed?
+ if saved_change_to_trash_at? or saved_change_to_owner_uuid?
# The group was added or removed from the trash.
#
# Strategy:
end
def after_ownership_change
- if owner_uuid_changed?
+ if saved_change_to_owner_uuid?
update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
end
end
protected
+ def check_readable_uuid attr, attr_value
+ if attr == 'tail_uuid' &&
+ !attr_value.nil? &&
+ self.link_class == 'permission' &&
+ attr_value[0..4] != Rails.configuration.ClusterID &&
+ ApiClientAuthorization.remote_host(uuid_prefix: attr_value[0..4]) &&
+ ArvadosModel::resource_class_for_uuid(attr_value) == User
+ # Permission link tail is a remote user (the user permissions
+ # are being granted to), so bypass the standard check that a
+ # referenced object uuid is readable by current user.
+ #
+ # We could do a call to the remote cluster to check if the user
+ # in tail_uuid exists. This would detect copy-and-paste errors,
+ # but add another way for the request to fail, and I don't think
+ # it would improve security. It doesn't seem to be worth the
+ # complexity tradeoff.
+ true
+ else
+ super
+ end
+ end
+
def permission_to_attach_to_objects
# Anonymous users cannot write links
return false if !current_user
head_obj = ArvadosModel.find_by_uuid(head_uuid)
+ if head_obj.nil?
+ errors.add(:head_uuid, "does not exist")
+ return false
+ end
+
# No permission links can be pointed to past collection versions
if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
errors.add(:head_uuid, "cannot point to a past version of a collection")
end
def dns_server_update
- if ip_address_changed? && ip_address
+ if saved_change_to_ip_address? && ip_address
Node.where('id != ? and ip_address = ?',
id, ip_address).each do |stale_node|
# One or more(!) stale node records have the same IP address
stale_node.update_attributes!(ip_address: nil)
end
end
- if hostname_was && hostname_changed?
- self.class.dns_server_update(hostname_was, UNUSED_NODE_IP)
+ if hostname_before_last_save && saved_change_to_hostname?
+ self.class.dns_server_update(hostname_before_last_save, UNUSED_NODE_IP)
end
- if hostname && (hostname_changed? || ip_address_changed?)
+ if hostname && (saved_change_to_hostname? || saved_change_to_ip_address?)
self.class.dns_server_update(hostname, ip_address || UNUSED_NODE_IP)
end
end
validate :must_unsetup_to_deactivate
before_update :prevent_privilege_escalation
before_update :prevent_inactive_admin
- before_update :verify_repositories_empty, :if => Proc.new { |user|
- user.username.nil? and user.username_changed?
+ before_update :verify_repositories_empty, :if => Proc.new {
+ username.nil? and username_changed?
}
before_update :setup_on_activate
before_create :check_auto_admin
- before_create :set_initial_username, :if => Proc.new { |user|
- user.username.nil? and user.email
+ before_create :set_initial_username, :if => Proc.new {
+ username.nil? and email
}
after_create :after_ownership_change
after_create :setup_on_activate
after_create :add_system_group_permission_link
- after_create :auto_setup_new_user, :if => Proc.new { |user|
+ after_create :auto_setup_new_user, :if => Proc.new {
Rails.configuration.Users.AutoSetupNewUsers and
- (user.uuid != system_user_uuid) and
- (user.uuid != anonymous_user_uuid)
+ (uuid != system_user_uuid) and
+ (uuid != anonymous_user_uuid)
}
after_create :send_admin_notifications
before_update :before_ownership_change
after_update :after_ownership_change
after_update :send_profile_created_notification
- after_update :sync_repository_names, :if => Proc.new { |user|
- (user.uuid != system_user_uuid) and
- user.username_changed? and
- (not user.username_was.nil?)
+ after_update :sync_repository_names, :if => Proc.new {
+ (uuid != system_user_uuid) and
+ saved_change_to_username? and
+ (not username_before_last_save.nil?)
}
before_destroy :clear_permissions
after_destroy :remove_self_from_permissions
end
def after_ownership_change
- if owner_uuid_changed?
+ if saved_change_to_owner_uuid?
update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
end
end
name: 'can_login').destroy_all
# delete "All users" group read permissions for this user
- group = Group.where(name: 'All users').select do |g|
- g[:uuid].match(/-f+$/)
- end.first
Link.where(tail_uuid: self.uuid,
- head_uuid: group[:uuid],
+ head_uuid: all_users_group_uuid,
link_class: 'permission',
name: 'can_read').destroy_all
self.is_active_was &&
!self.is_active
- group = Group.where(name: 'All users').select do |g|
- g[:uuid].match(/-f+$/)
- end.first
-
# When a user is set up, they are added to the "All users"
# group. A user that is part of the "All users" group is
# allowed to self-activate.
# explaining the correct way to deactivate a user.
#
if Link.where(tail_uuid: self.uuid,
- head_uuid: group[:uuid],
+ head_uuid: all_users_group_uuid,
link_class: 'permission',
name: 'can_read').any?
errors.add :is_active, "cannot be set to false directly, use the 'Deactivate' button on Workbench, or the 'unsetup' API call"
# add the user to the 'All users' group
def create_user_group_link
return (Link.where(tail_uuid: self.uuid,
- head_uuid: all_users_group[:uuid],
+ head_uuid: all_users_group_uuid,
link_class: 'permission',
name: 'can_read').first or
Link.create(tail_uuid: self.uuid,
- head_uuid: all_users_group[:uuid],
+ head_uuid: all_users_group_uuid,
link_class: 'permission',
name: 'can_read'))
end
# Automatically setup if is_active flag turns on
def setup_on_activate
return if [system_user_uuid, anonymous_user_uuid].include?(self.uuid)
- if is_active && (new_record? || is_active_changed?)
+ if is_active &&
+ (new_record? || saved_change_to_is_active? || will_save_change_to_is_active?)
setup
end
end
# Send notification if the user saved profile for the first time
def send_profile_created_notification
- if self.prefs_changed?
- if self.prefs_was.andand.empty? || !self.prefs_was.andand['profile']
+ if saved_change_to_prefs?
+ if prefs_before_last_save.andand.empty? || !prefs_before_last_save.andand['profile']
profile_notification_address = Rails.configuration.Users.UserProfileNotificationAddress
ProfileNotifier.profile_created(self, profile_notification_address).deliver_now if profile_notification_address and !profile_notification_address.empty?
end
end
def sync_repository_names
- old_name_re = /^#{Regexp.escape(username_was)}\//
+ old_name_re = /^#{Regexp.escape(username_before_last_save)}\//
name_sub = "#{username}/"
repositories.find_each do |repo|
repo.name = repo.name.sub(old_name_re, name_sub)
#
# SPDX-License-Identifier: AGPL-3.0
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
load Gem.bin_path('bundler', 'bundle')
#
# SPDX-License-Identifier: AGPL-3.0
-require 'pathname'
require 'fileutils'
include FileUtils
# path to your application root.
-APP_ROOT = Pathname.new File.expand_path('../../', __FILE__)
+APP_ROOT = File.expand_path('..', __dir__)
def system!(*args)
system(*args) || abort("\n== Command #{args} failed ==")
#
# SPDX-License-Identifier: AGPL-3.0
-require 'pathname'
require 'fileutils'
include FileUtils
# path to your application root.
-APP_ROOT = Pathname.new File.expand_path('../../', __FILE__)
+APP_ROOT = File.expand_path('..', __dir__)
def system!(*args)
system(*args) || abort("\n== Command #{args} failed ==")
--- /dev/null
+#!/usr/bin/env ruby
+
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+APP_ROOT = File.expand_path('..', __dir__)
+Dir.chdir(APP_ROOT) do
+ begin
+ exec "yarnpkg", *ARGV
+ rescue Errno::ENOENT
+ $stderr.puts "Yarn executable was not detected in the system."
+ $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install"
+ exit 1
+ end
+end
#
# SPDX-License-Identifier: AGPL-3.0
-require File.expand_path('../boot', __FILE__)
+require_relative 'boot'
require "rails"
# Pick only the frameworks we need:
require "action_controller/railtie"
require "action_mailer/railtie"
require "action_view/railtie"
-# Skip ActionCable (new in Rails 5.0) as it adds '/cable' routes that we're not using
-# require "action_cable/engine"
require "sprockets/railtie"
require "rails/test_unit/railtie"
+# Skipping the following:
+# * ActionCable (new in Rails 5.0) as it adds '/cable' routes that we're not using
+# * Skip ActiveStorage (new in Rails 5.1)
require 'digest'
# Set up gems listed in the Gemfile.
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
-require 'bundler/setup'
+require 'bundler/setup' # Set up gems listed in the Gemfile.
+require 'bootsnap/setup' # Speed up boot time by caching expensive operations.
\ No newline at end of file
#
# SPDX-License-Identifier: AGPL-3.0
-Server::Application.configure do
+Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb
# In the development environment your application's code is reloaded on
#
# SPDX-License-Identifier: AGPL-3.0
-Server::Application.configure do
+Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb
# Code is not reloaded between requests
#
# SPDX-License-Identifier: AGPL-3.0
-Server::Application.configure do
+Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb
# The test environment is used exclusively to run your application's
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Be sure to restart your server when you modify this file.
+
+# Define an application-wide content security policy
+# For further information see the following documentation
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
+
+# Rails.application.config.content_security_policy do |policy|
+# policy.default_src :self, :https
+# policy.font_src :self, :https, :data
+# policy.img_src :self, :https, :data
+# policy.object_src :none
+# policy.script_src :self, :https
+# policy.style_src :self, :https
+
+# # Specify URI for violation reports
+# # policy.report_uri "/csp-violation-report-endpoint"
+# end
+
+# If you are using UJS then enable automatic nonce generation
+# Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) }
+
+# Report CSP violations to a specified URI
+# For further information see the following documentation:
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only
+# Rails.application.config.content_security_policy_report_only = true
require 'enable_jobs_api'
-Server::Application.configure do
- if ActiveRecord::Base.connection.tables.include?('jobs')
- check_enable_legacy_jobs_api
+Rails.application.configure do
+ begin
+ if ActiveRecord::Base.connection.tables.include?('jobs')
+ check_enable_legacy_jobs_api
+ end
+ rescue ActiveRecord::NoDatabaseError
+ # Since rails 5.2, all initializers are run by rake tasks (like db:create),
+ # see: https://github.com/rails/rails/issues/32870
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Be sure to restart your server when you modify this file.
+#
+# This file contains migration options to ease your Rails 5.2 upgrade.
+#
+# Once upgraded flip defaults one by one to migrate to the new default.
+#
+# Read the Guide for Upgrading Ruby on Rails for more info on each option.
+
+# Make Active Record use stable #cache_key alongside new #cache_version method.
+# This is needed for recyclable cache keys.
+# Rails.application.config.active_record.cache_versioning = true
+
+# Use AES-256-GCM authenticated encryption for encrypted cookies.
+# Also, embed cookie expiry in signed or encrypted cookies for increased security.
+#
+# This option is not backwards compatible with earlier Rails versions.
+# It's best enabled when your entire app is migrated and stable on 5.2.
+#
+# Existing cookies will be converted on read then written with the new scheme.
+# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true
+
+# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages
+# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true.
+# Rails.application.config.active_support.use_authenticated_message_encryption = true
+
+# Add default protection from forgery to ActionController::Base instead of in
+# ApplicationController.
+# Rails.application.config.action_controller.default_protect_from_forgery = true
+
+# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and
+# 'f' after migrating old data.
+# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
+
+# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header.
+# Rails.application.config.active_support.use_sha1_digests = true
+
+# Make `form_with` generate id attributes for any generated HTML tags.
+# Rails.application.config.action_view.form_with_generates_ids = true
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# See http://aaronvb.com/articles/37-rails-caching-and-undefined-class-module
-
-# Config must be done before we load model class files; otherwise they
-# won't be able to use Rails.configuration.* to initialize their
-# classes.
-
-if Rails.env == 'development'
- Dir.foreach("#{Rails.root}/app/models") do |model_file|
- require_dependency model_file if model_file.match(/\.rb$/)
- end
-end
#
# SPDX-License-Identifier: AGPL-3.0
-ActiveRecord::Base.connection.class.set_callback :checkout, :after do
+ActiveRecord::ConnectionAdapters::AbstractAdapter.set_callback :checkout, :before, ->(conn) do
# If the database connection is in a time zone other than UTC,
# "timestamp" values don't behave as desired.
#
# before now()), but false in time zone -0100 (now() returns an
# earlier clock time, and its time zone is dropped when comparing to
# a "timestamp without time zone").
- raw_connection.sync_exec("SET TIME ZONE 'UTC'")
+ conn.execute("SET TIME ZONE 'UTC'")
end
# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
ActiveSupport.on_load(:action_controller) do
- wrap_parameters :format => [:json]
+ wrap_parameters format: [:json]
end
# Disable root element in JSON by default.
#
# SPDX-License-Identifier: AGPL-3.0
-Server::Application.routes.draw do
+Rails.application.routes.draw do
themes_for_rails
# OPTIONS requests are not allowed at routes that use cookies.
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Be sure to restart your server when you modify this file.
+
+# Your secret key is used for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+# You can use `rails secret` to generate a secure secret key.
+
+# NOTE that these get overriden by Arvados' own configuration system.
+
+# shared:
+# api_key: a1B2c3D4e5F6
+
+# Environmental secrets are only available for that specific environment.
+
+# development:
+# secret_key_base: <%= rand(1<<255).to_s(36) %>
+
+# test:
+# secret_key_base: <%= rand(1<<255).to_s(36) %>
+
+# In case this doesn't get overriden for some reason, assign a random key
+# to gracefully degrade by rejecting cookies instead of by opening a
+# vulnerability.
+production:
+ secret_key_base: <%= rand(1<<255).to_s(36) %>
rescue => e
Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
ensure
- ActiveRecord::Base.connection.close
+ # Rails 5.1+ makes test threads share a database connection, so we can't
+ # close a connection shared with other threads.
+ # https://github.com/rails/rails/commit/deba47799ff905f778e0c98a015789a1327d5087
+ if Rails.env != "test"
+ ActiveRecord::Base.connection.close
+ end
end
end
end
rescue => e
Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
ensure
- ActiveRecord::Base.connection.close
+ # Rails 5.1+ makes test threads share a database connection, so we can't
+ # close a connection shared with other threads.
+ # https://github.com/rails/rails/commit/deba47799ff905f778e0c98a015789a1327d5087
+ if Rails.env != "test"
+ ActiveRecord::Base.connection.close
+ end
end
end
end
# priority==0 but should be >0:
act_as_system_user do
Container.
- joins("JOIN container_requests ON container_requests.container_uuid=containers.uuid AND container_requests.state=#{Container.sanitize(ContainerRequest::Committed)} AND container_requests.priority>0").
+ joins("JOIN container_requests ON container_requests.container_uuid=containers.uuid AND container_requests.state=#{ActiveRecord::Base.connection.quote(ContainerRequest::Committed)} AND container_requests.priority>0").
where('containers.state IN (?) AND containers.priority=0 AND container_requests.uuid IS NOT NULL',
[Container::Queued, Container::Locked, Container::Running]).
map(&:update_priority!)
rescue => e
Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
ensure
- ActiveRecord::Base.connection.close
+ # Rails 5.1+ makes test threads share a database connection, so we can't
+ # close a connection shared with other threads.
+ # https://github.com/rails/rails/commit/deba47799ff905f778e0c98a015789a1327d5087
+ if Rails.env != "test"
+ ActiveRecord::Base.connection.close
+ end
end
end
end
refute_empty expect_rvz
authorize_with :active
get :index,
- params: {:format => :json},
- headers: auth(:active)
+ params: {:format => :json}
assert_response :success
json_response['items'].each do |svc|
url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}/"
c.reload
assert_equal({'foo' => 'bar'}, c.properties)
end
+
+ test 'serialized attributes dirty tracking with audit log settings' do
+ Rails.configuration.AuditLogs.MaxDeleteBatch = 1000
+ set_user_from_auth :admin
+ [false, true].each do |auditlogs_enabled|
+ if auditlogs_enabled
+ Rails.configuration.AuditLogs.MaxAge = 3600
+ else
+ Rails.configuration.AuditLogs.MaxAge = 0
+ end
+ [
+ User.find_by_uuid(users(:active).uuid),
+ ContainerRequest.find_by_uuid(container_requests(:queued).uuid),
+ Container.find_by_uuid(containers(:queued).uuid),
+ PipelineInstance.find_by_uuid(pipeline_instances(:has_component_with_completed_jobs).uuid),
+ PipelineTemplate.find_by_uuid(pipeline_templates(:two_part).uuid),
+ Job.find_by_uuid(jobs(:running).uuid)
+ ].each do |obj|
+ assert_not(obj.class.serialized_attributes.empty?,
+ "#{obj.class} model doesn't have serialized attributes")
+ # obj shouldn't have changed since it's just retrieved from the database
+ assert_not(obj.changed?, "#{obj.class} model's attribute(s) appear as changed: '#{obj.changes.keys.join(',')}' with audit logs #{auditlogs_enabled ? '': 'not '}enabled.")
+ end
+ end
+ end
end
users(:active).uuid.sub(/-\w+$/, "-#{'z' * 15}"))
end
+ test "link granting permission to remote user is valid" do
+ refute new_active_link_valid?(tail_uuid:
+ users(:active).uuid.sub(/^\w+-/, "foooo-"))
+ Rails.configuration.RemoteClusters = Rails.configuration.RemoteClusters.merge({foooo: ActiveSupport::InheritableOptions.new({Host: "bar.com"})})
+ assert new_active_link_valid?(tail_uuid:
+ users(:active).uuid.sub(/^\w+-/, "foooo-"))
+ end
+
test "link granting non-project permission to unreadable user is invalid" do
refute new_active_link_valid?(tail_uuid: users(:admin).uuid,
head_uuid: collections(:bar_file).uuid)
sleep 0.1
end
assert_operator remaining_audit_logs.count, :<, initial_log_count
- ensure
- # The test framework rolls back our transactions, but that
- # doesn't undo the deletes we did from separate threads.
- ActiveRecord::Base.connection.exec_query 'ROLLBACK'
- Thread.new do
- begin
- dc = DatabaseController.new
- dc.define_singleton_method :render do |*args| end
- dc.reset
- ensure
- ActiveRecord::Base.connection.close
- end
- end.join
end
end
end
c.Fatal(err)
}
_, p, err := net.SplitHostPort(ln.Addr().String())
+ c.Check(err, check.IsNil)
ln.Close()
config := "Clusters:\n zzzzz:\n ManagementToken: abcdefg\n Services: {Keepbalance: {InternalURLs: {'http://localhost:" + p + "/': {}}}}\n"
+++ /dev/null
-#! /usr/bin/env python
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# traffic_test.py
-#
-# Launch a test Keep and API server and PUT and GET a bunch of blocks.
-# Can be used to simulate client traffic in Keep to evaluate memory usage,
-# error logging, performance, etc.
-#
-# This script is warty and is relatively environment-specific, but the
-# example run described below should execute cleanly.
-#
-# Usage:
-# traffic_test.py start
-# Starts the test servers.
-# traffic_test.py put file1 file2 file3 ....
-# Runs arv-put on each file.
-# traffic_test.py get hash1 hash2 hash3 ....
-# Loops forever issuing GET requests for specified blocks.
-# traffic_test.py stop
-# Stops the test servers.
-#
-# Example:
-#
-# $ ./traffic_test.py start
-# $ ./traffic_test.py put GS00253-DNA_A02_200_37.tsv.bz2 \
-# GS00253-DNA_B01_200_37.tsv.bz2 \
-# GS00253-DNA_B02_200_37.tsv.bz2
-# $ ./traffic_test.py get $(find /tmp/tmp* -type f -printf "%f ")
-# [loops forever]
-# ^C
-# $ ./traffic_test.py stop
-#
-# Multiple "get" runs may be run concurrently to evaluate Keep's handling
-# of additional concurrent clients.
-
-PYSDK_DIR = "../../../sdk/python"
-PYTEST_DIR = PYSDK_DIR + "/tests"
-ARV_PUT_PATH = PYSDK_DIR + "/bin/arv-put"
-ARV_GET_PATH = PYSDK_DIR + "/bin/arv-get"
-SECONDS_BETWEEN_GETS = 1
-
-import argparse
-import httplib2
-import os
-import random
-import subprocess
-import sys
-import time
-
-# for run_test_server.py
-sys.path.insert(0, PYSDK_DIR)
-sys.path.insert(0, PYTEST_DIR)
-import arvados
-import run_test_server
-
-def arv_cmd(*args):
- p = subprocess.Popen([sys.executable] + list(args),
- stdout=subprocess.PIPE)
- (arvout, arverr) = p.communicate()
- if p.returncode != 0:
- print "error {} from {} {}: {}".format(
- p.returncode, sys.executable, args, arverr)
- sys.exit(p.returncode)
- return arvout
-
-def start():
- run_test_server.run()
- run_test_server.run_keep()
-
-def put(files):
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
- run_test_server.authorize_with('active')
- for v in ["ARVADOS_API_HOST",
- "ARVADOS_API_HOST_INSECURE",
- "ARVADOS_API_TOKEN"]:
- os.environ[v] = arvados.config.settings()[v]
-
- if not os.environ.has_key('PYTHONPATH'):
- os.environ['PYTHONPATH'] = ''
- os.environ['PYTHONPATH'] = "{}:{}:{}".format(
- PYSDK_DIR, PYTEST_DIR, os.environ['PYTHONPATH'])
-
- for c in files:
- manifest_uuid = arv_cmd(ARV_PUT_PATH, c)
-
-def get(blocks):
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
-
- run_test_server.authorize_with('active')
- for v in ["ARVADOS_API_HOST",
- "ARVADOS_API_HOST_INSECURE",
- "ARVADOS_API_TOKEN"]:
- os.environ[v] = arvados.config.settings()[v]
-
- nqueries = 0
- while True:
- b = random.choice(blocks)
- print "GET /" + b
- body = arv_cmd(ARV_GET_PATH, b)
- print "got {} bytes".format(len(body))
- time.sleep(SECONDS_BETWEEN_GETS)
- nqueries = nqueries + 1
-
-def stop():
- run_test_server.stop_keep()
- run_test_server.stop()
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument('action',
- type=str,
- nargs='+',
- help='''"start", "put", "get", "stop"''')
- args = parser.parse_args()
-
- if args.action[0] == 'start':
- start()
- elif args.action[0] == 'put':
- put(args.action[1:])
- elif args.action[0] == 'get':
- get(args.action[1:])
- elif args.action[0] == 'stop':
- stop()
- else:
- print('Unrecognized action "{}"'.format(args.action))
- print('actions are "start", "put", "get", "stop"')
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err := v.check(s.metadata.URL + "/latest")
+ c.Check(err, check.IsNil)
creds, err := v.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
+ c.Check(err, check.IsNil)
c.Check(creds.AccessKeyID, check.Equals, "ASIAIOSFODNN7EXAMPLE")
c.Check(creds.SecretAccessKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
err = deadv.check(s.metadata.URL + "/latest")
+ c.Check(err, check.IsNil)
_, err = deadv.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
c.Check(err, check.ErrorMatches, `(?s).*EC2RoleRequestError: no EC2 instance role found.*`)
c.Check(err, check.ErrorMatches, `(?s).*404.*`)
projects / arvados.git / commitdiff