Thomas Mooney <tmooney@genome.wustl.edu>
Chen Chen <aflyhorse@gmail.com>
Veritas Genetics, Inc. <*@veritasgenetics.com>
-Curii Corporation <*@curii.com>
\ No newline at end of file
+Curii Corporation, Inc. <*@curii.com>
## Development
-[](https://ci.curoverse.com/job/run-tests/)
+[](https://ci.arvados.org/job/run-tests/)
[](https://goreportcard.com/report/github.com/arvados/arvados)
The Arvados public bug tracker is located at https://dev.arvados.org/projects/arvados/issues
end
def current_api_host
- "#{Rails.configuration.Services.Controller.ExternalURL.hostname}:#{Rails.configuration.Services.Controller.ExternalURL.port}"
+ if Rails.configuration.Services.Controller.ExternalURL.port == 443
+ "#{Rails.configuration.Services.Controller.ExternalURL.hostname}"
+ else
+ "#{Rails.configuration.Services.Controller.ExternalURL.hostname}:#{Rails.configuration.Services.Controller.ExternalURL.port}"
+ end
end
def current_uuid_prefix
<pre>
Host *.arvados
- ProxyCommand ssh -p2222 turnout@switchyard.<%= current_api_host || 'xyzzy.arvadosapi.com' %> -x -a $SSH_PROXY_FLAGS %h
+ ProxyCommand ssh -p2222 turnout@switchyard.<%= current_api_host.sub(/:.*$/,"") || 'xyzzy.arvadosapi.com' %> -x -a $SSH_PROXY_FLAGS %h
<% if @objects.first.andand.current_user_logins.andand.first %>
User <%= @objects.first.current_user_logins.andand.first %>
<% end %>
</p>
<pre>
-ssh <%= @objects.first.andand.hostname.andand.sub('.'+current_api_host,'') or 'vm-hostname' %>.arvados
+ssh <%= @objects.first.andand.hostname.andand.sub('.'+current_api_host.sub(/:.*$/,""),'') or 'vm-hostname' %>.arvados
</pre>
<p>
if cq.dontupdate != nil {
cq.dontupdate[uuid] = struct{}{}
}
- if ent, ok := cq.current[uuid]; !ok {
- cq.addEnt(uuid, resp)
- } else {
- ent.Container.State, ent.Container.Priority, ent.Container.LockedByUUID = resp.State, resp.Priority, resp.LockedByUUID
- cq.current[uuid] = ent
+ ent, ok := cq.current[uuid]
+ if !ok {
+ // Container is not in queue (e.g., it was not added
+ // because there is no suitable instance type, and
+ // we're just locking/updating it in order to set an
+ // error message). No need to add it, and we don't
+ // necessarily have enough information to add it here
+ // anyway because lock/unlock responses don't include
+ // runtime_constraints.
+ return
}
+ ent.Container.State, ent.Container.Priority, ent.Container.LockedByUUID = resp.State, resp.Priority, resp.LockedByUUID
+ cq.current[uuid] = ent
cq.notify()
}
client := arvados.NewClientFromEnv()
cq := NewQueue(logger(), nil, errorTypeChooser, client)
+ ch := cq.Subscribe()
+ go func() {
+ defer cq.Unsubscribe(ch)
+ for range ch {
+ // Container should never be added to
+ // queue. Note that polling the queue this way
+ // doesn't guarantee a bug (container being
+ // incorrectly added to the queue) will cause
+ // a test failure.
+ _, ok := cq.Get(arvadostest.QueuedContainerUUID)
+ if !c.Check(ok, check.Equals, false) {
+ // Don't spam the log with more failures
+ break
+ }
+ }
+ }()
+
var ctr arvados.Container
err := client.RequestAndDecode(&ctr, "GET", "arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil, nil)
c.Check(err, check.IsNil)
c.Check(ctr.State, check.Equals, arvados.ContainerStateQueued)
- cq.Update()
-
// Wait for the cancel operation to take effect. Container
// will have state=Cancelled or just disappear from the queue.
suite.waitfor(c, time.Second, func() bool {
+ cq.Update()
err := client.RequestAndDecode(&ctr, "GET", "arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil, nil)
return err == nil && ctr.State == arvados.ContainerStateCancelled
})
// A new seg.buf has been allocated.
return
}
- seg.flushing = nil
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
offsets := make([]int, 0, len(refs)) // location of segment's data within block
for _, ref := range refs {
seg := ref.fn.segments[ref.idx].(*memSegment)
- if seg.flushing != nil && !sync {
+ if !sync && seg.flushingUnfinished() {
// Let the other flushing goroutine finish. If
// it fails, we'll try again next time.
+ close(done)
return nil
} else {
// In sync mode, we proceed regardless of
defer close(errs)
locator, _, err := dn.fs.PutB(block)
dn.fs.throttle().Release()
- {
- defer func() {
- for _, seg := range segs {
- if seg.flushing == done {
- seg.flushing = nil
- }
- }
- }()
- }
if err != nil {
errs <- err
return
type memSegment struct {
buf []byte
- // If flushing is not nil, then a) buf is being shared by a
- // pruneMemSegments goroutine, and must be copied on write;
- // and b) the flushing channel will close when the goroutine
- // finishes, whether it succeeds or not.
+ // If flushing is not nil and not ready/closed, then a) buf is
+ // being shared by a pruneMemSegments goroutine, and must be
+ // copied on write; and b) the flushing channel will close
+ // when the goroutine finishes, whether it succeeds or not.
flushing <-chan struct{}
}
+func (me *memSegment) flushingUnfinished() bool {
+ if me.flushing == nil {
+ return false
+ }
+ select {
+ case <-me.flushing:
+ me.flushing = nil
+ return false
+ default:
+ return true
+ }
+}
+
func (me *memSegment) Len() int {
return len(me.buf)
}
"os"
"regexp"
"runtime"
+ "runtime/pprof"
"strings"
"sync"
"sync/atomic"
fs.Flush("", true)
}
+func (s *CollectionFSSuite) TestFlushStress(c *check.C) {
+ done := false
+ defer func() { done = true }()
+ time.AfterFunc(10*time.Second, func() {
+ if !done {
+ pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
+ panic("timeout")
+ }
+ })
+
+ wrote := 0
+ s.kc.onPut = func(p []byte) {
+ s.kc.Lock()
+ s.kc.blocks = map[string][]byte{}
+ wrote++
+ defer c.Logf("wrote block %d, %d bytes", wrote, len(p))
+ s.kc.Unlock()
+ time.Sleep(20 * time.Millisecond)
+ }
+
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ data := make([]byte, 1<<20)
+ for i := 0; i < 3; i++ {
+ dir := fmt.Sprintf("dir%d", i)
+ fs.Mkdir(dir, 0755)
+ for j := 0; j < 200; j++ {
+ data[0] = byte(j)
+ f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write(data)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ fs.Flush(dir, false)
+ }
+ _, err := fs.MarshalManifest(".")
+ c.Check(err, check.IsNil)
+ }
+}
+
func (s *CollectionFSSuite) TestFlushShort(c *check.C) {
s.kc.onPut = func([]byte) {
s.kc.Lock()
f.Close()
fs.Flush(dir, false)
}
+ fs.Flush(dir, true)
+ _, err := fs.MarshalManifest(".")
+ c.Check(err, check.IsNil)
}
}
raise ArgumentError.new "Required uuid or user"
elsif !params[:user]['email']
raise ArgumentError.new "Require user email"
- elsif !params[:openid_prefix]
- raise ArgumentError.new "Required openid_prefix parameter is missing."
else
@object = model_class.create! resource_attrs
end
end
@response = @object.setup(repo_name: full_repo_name,
- vm_uuid: params[:vm_uuid],
- openid_prefix: params[:openid_prefix])
+ vm_uuid: params[:vm_uuid])
# setup succeeded. send email to user
if params[:send_notification_email]
def self._setup_requires_parameters
{
+ uuid: {
+ type: 'string', required: false
+ },
user: {
type: 'object', required: false
},
- openid_prefix: {
- type: 'string', required: false
- },
repo_name: {
type: 'string', required: false
},
end
# create links
- def setup(openid_prefix:, repo_name: nil, vm_uuid: nil)
+ def setup(repo_name: nil, vm_uuid: nil)
repo_perm = create_user_repo_link repo_name
vm_login_perm = create_vm_login_permission_link(vm_uuid, username) if vm_uuid
group_perm = create_user_group_link
def setup_on_activate
return if [system_user_uuid, anonymous_user_uuid].include?(self.uuid)
if is_active && (new_record? || is_active_changed?)
- setup(openid_prefix: Rails.configuration.default_openid_prefix)
+ setup
end
end
# Automatically setup new user during creation
def auto_setup_new_user
- setup(openid_prefix: Rails.configuration.default_openid_prefix)
+ setup
if username
create_vm_login_permission_link(Rails.configuration.Users.AutoSetupNewUsersWithVmUUID,
username)
post :setup, params: {
repo_name: repo_name,
- openid_prefix: 'https://www.google.com/accounts/o8/id',
user: {
uuid: 'zzzzz-tpzed-abcdefghijklmno',
first_name: "in_create_test_first_name",
user: {uuid: 'bogus_uuid'},
repo_name: 'usertestrepo',
vm_uuid: @vm_uuid,
- openid_prefix: 'https://www.google.com/accounts/o8/id'
}
response_body = JSON.parse(@response.body)
response_errors = response_body['errors']
post :setup, params: {
repo_name: 'usertestrepo',
vm_uuid: @vm_uuid,
- openid_prefix: 'https://www.google.com/accounts/o8/id'
}
response_body = JSON.parse(@response.body)
response_errors = response_body['errors']
user: {},
repo_name: 'usertestrepo',
vm_uuid: @vm_uuid,
- openid_prefix: 'https://www.google.com/accounts/o8/id'
}
response_body = JSON.parse(@response.body)
response_errors = response_body['errors']
post :setup, params: {
repo_name: 'usertestrepo',
user: {email: 'foo@example.com'},
- openid_prefix: 'https://www.google.com/accounts/o8/id'
}
assert_response :success
repo_name: 'usertestrepo',
vm_uuid: 'no_such_vm',
user: {email: 'foo@example.com'},
- openid_prefix: 'https://www.google.com/accounts/o8/id'
}
response_body = JSON.parse(@response.body)
post :setup, params: {
repo_name: 'usertestrepo',
- openid_prefix: 'https://www.google.com/accounts/o8/id',
vm_uuid: @vm_uuid,
user: {email: 'foo@example.com'}
}
post :setup, params: {
user: {email: 'foo@example.com'},
- openid_prefix: 'https://www.google.com/accounts/o8/id'
}
assert_response :success
authorize_with :admin
post :setup, params: {
- openid_prefix: 'https://www.google.com/accounts/o8/id',
repo_name: 'usertestrepo',
vm_uuid: @vm_uuid,
user: {
inactive_user = users(:inactive)
post :setup, params: {
- openid_prefix: 'https://www.google.com/accounts/o8/id',
repo_name: 'usertestrepo',
user: {
email: inactive_user['email']
post :setup, params: {
repo_name: 'usertestrepo',
- openid_prefix: 'http://www.example.com/account',
user: {
first_name: "in_create_test_first_name",
last_name: "test_last_name",
nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
end
- test "invoke setup with no openid prefix, expect error" do
- authorize_with :admin
-
- post :setup, params: {
- repo_name: 'usertestrepo',
- user: {
- first_name: "in_create_test_first_name",
- last_name: "test_last_name",
- email: "foo@example.com"
- }
- }
-
- response_body = JSON.parse(@response.body)
- response_errors = response_body['errors']
- assert_not_nil response_errors, 'Expected error in response'
- assert (response_errors.first.include? 'openid_prefix parameter is missing'),
- 'Expected ArgumentError'
- end
-
test "setup user with user, vm and repo and verify links" do
authorize_with :admin
},
vm_uuid: @vm_uuid,
repo_name: 'usertestrepo',
- openid_prefix: 'https://www.google.com/accounts/o8/id'
}
assert_response :success
authorize_with :active
post :setup, params: {
- openid_prefix: 'https://www.google.com/accounts/o8/id',
user: {email: 'foo@example.com'}
}
authorize_with :admin
post :setup, params: {
- openid_prefix: 'http://www.example.com/account',
send_notification_email: 'false',
user: {
email: "foo@example.com"
authorize_with :admin
post :setup, params: {
- openid_prefix: 'http://www.example.com/account',
send_notification_email: 'true',
user: {
email: "foo@example.com"
post "/arvados/v1/users/setup",
params: {
repo_name: repo_name,
- openid_prefix: 'https://www.google.com/accounts/o8/id',
user: {
uuid: 'zzzzz-tpzed-abcdefghijklmno',
first_name: "in_create_test_first_name",
params: {
repo_name: repo_name,
vm_uuid: virtual_machines(:testvm).uuid,
- openid_prefix: 'https://www.google.com/accounts/o8/id',
user: {
uuid: 'zzzzz-tpzed-abcdefghijklmno',
first_name: "in_create_test_first_name",
params: {
repo_name: repo_name,
vm_uuid: virtual_machines(:testvm).uuid,
- openid_prefix: 'https://www.google.com/accounts/o8/id',
uuid: 'zzzzz-tpzed-abcdefghijklmno',
},
headers: auth(:admin)
test "setup user in multiple steps and verify response" do
post "/arvados/v1/users/setup",
params: {
- openid_prefix: 'http://www.example.com/account',
user: {
email: "foo@example.com"
}
# invoke setup with a repository
post "/arvados/v1/users/setup",
params: {
- openid_prefix: 'http://www.example.com/account',
repo_name: 'newusertestrepo',
uuid: created['uuid']
},
post "/arvados/v1/users/setup",
params: {
vm_uuid: virtual_machines(:testvm).uuid,
- openid_prefix: 'http://www.example.com/account',
user: {
email: 'junk_email'
},
repo_name: 'newusertestrepo',
vm_uuid: virtual_machines(:testvm).uuid,
user: {email: 'foo@example.com'},
- openid_prefix: 'https://www.google.com/accounts/o8/id'
},
headers: auth(:admin)
set_user_from_auth :admin
email = 'foo@example.com'
- openid_prefix = 'http://openid/prefix'
user = User.create ({uuid: 'zzzzz-tpzed-abcdefghijklmno', email: email})
vm = VirtualMachine.create
- response = user.setup(openid_prefix: openid_prefix,
- repo_name: 'foo/testrepo',
+ response = user.setup(repo_name: 'foo/testrepo',
vm_uuid: vm.uuid)
resp_user = find_obj_in_resp response, 'User'
set_user_from_auth :admin
email = 'foo@example.com'
- openid_prefix = 'http://openid/prefix'
user = User.create ({uuid: 'zzzzz-tpzed-abcdefghijklmno', email: email})
verify_link resp_link, 'permission', 'can_login', email, bad_uuid
- response = user.setup(openid_prefix: openid_prefix,
- repo_name: 'foo/testrepo',
+ response = user.setup(repo_name: 'foo/testrepo',
vm_uuid: vm.uuid)
resp_user = find_obj_in_resp response, 'User'
set_user_from_auth :admin
email = 'foo@example.com'
- openid_prefix = 'http://openid/prefix'
user = User.create ({uuid: 'zzzzz-tpzed-abcdefghijklmno', email: email})
- response = user.setup(openid_prefix: openid_prefix)
+ response = user.setup()
resp_user = find_obj_in_resp response, 'User'
verify_user resp_user, email
verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
# invoke setup again with repo_name
- response = user.setup(openid_prefix: openid_prefix,
- repo_name: 'foo/testrepo')
+ response = user.setup(repo_name: 'foo/testrepo')
resp_user = find_obj_in_resp response, 'User', nil
verify_user resp_user, email
assert_equal user.uuid, resp_user[:uuid], 'expected uuid not found'
# invoke setup again with a vm_uuid
vm = VirtualMachine.create
- response = user.setup(openid_prefix: openid_prefix,
- repo_name: 'foo/testrepo',
+ response = user.setup(repo_name: 'foo/testrepo',
vm_uuid: vm.uuid)
resp_user = find_obj_in_resp response, 'User', nil
projects / arvados.git / commitdiff