// environment variables ARVADOS_API_HOST, ARVADOS_API_TOKEN,
// ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT, and
// ARVADOS_KEEP_SERVICES.
-func MakeArvadosClient() (ac ArvadosClient, err error) {
+func MakeArvadosClient() (ac *ArvadosClient, err error) {
var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
- ac = ArvadosClient{
+ ac = &ArvadosClient{
Scheme: "https",
ApiServer: os.Getenv("ARVADOS_API_HOST"),
ApiToken: os.Getenv("ARVADOS_API_TOKEN"),
// CallRaw is the same as Call() but returns a Reader that reads the
// response body, instead of taking an output object.
-func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) {
+func (c *ArvadosClient) CallRaw(method string, resourceType string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) {
scheme := c.Scheme
if scheme == "" {
scheme = "https"
// Returns a non-nil error if an error occurs making the API call, the
// API responds with a non-successful HTTP status, or an error occurs
// parsing the response body.
-func (c ArvadosClient) Call(method, resourceType, uuid, action string, parameters Dict, output interface{}) error {
+func (c *ArvadosClient) Call(method, resourceType, uuid, action string, parameters Dict, output interface{}) error {
reader, err := c.CallRaw(method, resourceType, uuid, action, parameters)
if reader != nil {
defer reader.Close()
}
// Create a new resource. See Call for argument descriptions.
-func (c ArvadosClient) Create(resourceType string, parameters Dict, output interface{}) error {
+func (c *ArvadosClient) Create(resourceType string, parameters Dict, output interface{}) error {
return c.Call("POST", resourceType, "", "", parameters, output)
}
// Delete a resource. See Call for argument descriptions.
-func (c ArvadosClient) Delete(resource string, uuid string, parameters Dict, output interface{}) (err error) {
+func (c *ArvadosClient) Delete(resource string, uuid string, parameters Dict, output interface{}) (err error) {
return c.Call("DELETE", resource, uuid, "", parameters, output)
}
// Modify attributes of a resource. See Call for argument descriptions.
-func (c ArvadosClient) Update(resourceType string, uuid string, parameters Dict, output interface{}) (err error) {
+func (c *ArvadosClient) Update(resourceType string, uuid string, parameters Dict, output interface{}) (err error) {
return c.Call("PUT", resourceType, uuid, "", parameters, output)
}
// Get a resource. See Call for argument descriptions.
-func (c ArvadosClient) Get(resourceType string, uuid string, parameters Dict, output interface{}) (err error) {
+func (c *ArvadosClient) Get(resourceType string, uuid string, parameters Dict, output interface{}) (err error) {
if !UUIDMatch(uuid) && !(resourceType == "collections" && PDHMatch(uuid)) {
// No object has uuid == "": there is no need to make
// an API call. Furthermore, the HTTP request for such
}
// List resources of a given type. See Call for argument descriptions.
-func (c ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) {
+func (c *ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) {
return c.Call("GET", resource, "", "", parameters, output)
}
func MakeClientPool() *ClientPool {
proto, err := MakeArvadosClient()
return &ClientPool{
- Prototype: &proto,
+ Prototype: proto,
lastErr: err,
}
}
}
var kc IKeepClient
- kc, err = keepclient.MakeKeepClient(&api)
+ kc, err = keepclient.MakeKeepClient(api)
if err != nil {
log.Fatal(err)
}
// Dispatcher holds the state of the dispatcher
type Dispatcher struct {
// The Arvados client
- Arv arvadosclient.ArvadosClient
+ Arv *arvadosclient.ArvadosClient
// When a new queued container appears and is either already owned by
// this dispatcher or is successfully locked, the dispatcher will call
var _ = check.Suite(&CollectionReaderUnit{})
type CollectionReaderUnit struct {
- arv arvadosclient.ArvadosClient
+ arv *arvadosclient.ArvadosClient
kc *KeepClient
handler SuccessHandler
}
c.Assert(err, check.IsNil)
s.arv.ApiToken = arvadostest.ActiveToken
- s.kc, err = MakeKeepClient(&s.arv)
+ s.kc, err = MakeKeepClient(s.arv)
c.Assert(err, check.IsNil)
s.handler = SuccessHandler{
if err != nil {
panic(err)
}
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
if err != nil {
panic(err)
}
// arv2 should use our stub servers, but one created for arv1
// should not.
- kc1, err := MakeKeepClient(&arv1)
+ kc1, err := MakeKeepClient(arv1)
c.Assert(err, check.IsNil)
- kc2, err := MakeKeepClient(&arv2)
+ kc2, err := MakeKeepClient(arv2)
c.Assert(err, check.IsNil)
_, _, _, err = kc1.Get(hash)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
c.Check(len(kc.LocalRoots()), Equals, 2)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(kc.Want_replicas, Equals, 2)
arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
- kc, err = MakeKeepClient(&arv)
+ kc, err = MakeKeepClient(arv)
c.Assert(kc.Want_replicas, Equals, 3)
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
- kc, err = MakeKeepClient(&arv)
+ kc, err = MakeKeepClient(arv)
c.Assert(kc.Want_replicas, Equals, 1)
}
arv, _ := arvadosclient.MakeArvadosClient()
arv.ApiToken = "abc123"
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
reader, writer := io.Pipe()
upload_status := make(chan uploadStatus)
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
make(chan string, 4)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
kc.Retries = 0
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
kc.Retries = 0
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{"x": ks0.url},
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{
defer ksGateway.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
content}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
writableLocalRoots := make(map[string]string)
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
hash := fmt.Sprintf("%x", md5.Sum(content))
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 3
arv.ApiToken = "abc123"
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
make(chan string, 5)}}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
// Make a keepclient and ensure that the testblobstore is included
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
// verify kc.LocalRoots
)
type LoggerParams struct {
- Client arvadosclient.ArvadosClient // The client we use to write log entries
- EventTypePrefix string // The prefix we use for the event type in the log entry
- WriteInterval time.Duration // Wait at least this long between log writes
+ Client *arvadosclient.ArvadosClient // The client we use to write log entries
+ EventTypePrefix string // The prefix we use for the event type in the log entry
+ WriteInterval time.Duration // Wait at least this long between log writes
}
// A LogMutator is a function which modifies the log entry.
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
)
-func UserIsAdmin(arv arvadosclient.ArvadosClient) (is_admin bool, err error) {
+func UserIsAdmin(arv *arvadosclient.ArvadosClient) (is_admin bool, err error) {
type user struct {
IsAdmin bool `json:"is_admin"`
}
// return
// count - the number of items of type resource the api server reports, if no error
// err - error accessing the resource, or nil if no error
-func NumberItemsAvailable(client arvadosclient.ArvadosClient, resource string) (count int, err error) {
+func NumberItemsAvailable(client *arvadosclient.ArvadosClient, resource string) (count int, err error) {
var response struct {
ItemsAvailable int `json:"items_available"`
}
include KindAndEtag
include CommonApiTemplate
include WhitelistUpdate
+ extend CurrentApiClient
serialize :environment, Hash
serialize :mounts, Hash
where('mounts = ?', self.deep_sort_hash(attrs[:mounts]).to_yaml).
where('runtime_constraints = ?', self.deep_sort_hash(attrs[:runtime_constraints]).to_yaml)
- # Check for Completed candidates that only had consistent outputs.
+ # Check for Completed candidates that had consistent outputs.
completed = candidates.where(state: Complete).where(exit_code: 0)
- if completed.select("output").group('output').limit(2).length == 1
- return completed.order('finished_at asc').limit(1).first
+ outputs = completed.select('output').group('output').limit(2)
+ if outputs.count.count != 1
+ Rails.logger.debug("Found #{outputs.count.length} different outputs")
+ elsif Collection.
+ readable_by(current_user).
+ where(portable_data_hash: outputs.first.output).
+ count < 1
+ Rails.logger.info("Found reusable container(s) " +
+ "but output #{outputs.first} is not readable " +
+ "by user #{current_user.uuid}")
+ else
+ # Return the oldest eligible container whose log is still
+ # present and readable by current_user.
+ readable_pdh = Collection.
+ readable_by(current_user).
+ select('portable_data_hash')
+ completed = completed.
+ where("log in (#{readable_pdh.to_sql})").
+ order('finished_at asc').
+ limit(1)
+ if completed.first
+ return completed.first
+ else
+ Rails.logger.info("Found reusable container(s) but none with a log " +
+ "readable by user #{current_user.uuid}")
+ end
end
# Check for Running candidates and return the most likely to finish sooner.
act_as_system_user do
# Notify container requests associated with this container
ContainerRequest.where(container_uuid: uuid,
- :state => ContainerRequest::Committed).each do |cr|
+ state: ContainerRequest::Committed).each do |cr|
cr.container_completed!
end
# Try to cancel any outstanding container requests made by this container.
ContainerRequest.where(requesting_container_uuid: uuid,
- :state => ContainerRequest::Committed).each do |cr|
+ state: ContainerRequest::Committed).each do |cr|
cr.priority = 0
cr.save
end
%w(modified_by_client_uuid container_uuid requesting_container_uuid)
end
+ # Finalize the container request after the container has
+ # finished/cancelled.
def container_completed!
- # may implement retry logic here in the future.
- self.state = ContainerRequest::Final
- self.save!
+ update_attributes!(state: ContainerRequest::Final)
+ c = Container.find_by_uuid(container_uuid)
+ ['output', 'log'].each do |out_type|
+ pdh = c.send(out_type)
+ next if pdh.nil?
+ manifest = Collection.where(portable_data_hash: pdh).first.manifest_text
+ Collection.create!(owner_uuid: owner_uuid,
+ manifest_text: manifest,
+ portable_data_hash: pdh,
+ name: "Container #{out_type} for request #{uuid}",
+ properties: {
+ 'type' => out_type,
+ 'container_request' => uuid,
+ })
+ end
end
protected
container_image: test
cwd: test
log: ea10d51bcf88862dbcc36eb292017dfd+45
- output: zzzzz-4zz18-znfnqtbbv4spc3w
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
output_path: test
command: ["echo", "hello"]
runtime_constraints:
finished_at: 2016-01-14 11:12:13.111111111 Z
container_image: test
cwd: test
- output: test
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
output_path: test
command: ["echo", "hello"]
runtime_constraints:
updated_at: 2016-01-11 11:11:11.111111111 Z
container_image: test
cwd: test
- output: test
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
output_path: test
command: ["echo", "hello"]
runtime_constraints:
updated_at: 2016-01-11 11:11:11.111111111 Z
container_image: test
cwd: test
- output: test
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
output_path: test
command: ["echo", "hello"]
runtime_constraints:
updated_at: 2016-01-11 11:11:11.111111111 Z
container_image: test
cwd: test
- output: test
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
output_path: test
command: ["echo", "hello"]
runtime_constraints:
test "Request is finalized when its container is completed" do
set_user_from_auth :active
- cr = create_minimal_req!(priority: 1, state: "Committed")
+ project = groups(:private)
+ cr = create_minimal_req!(owner_uuid: project.uuid,
+ priority: 1,
+ state: "Committed")
c = act_as_system_user do
c = Container.find_by_uuid(cr.container_uuid)
assert_equal "Committed", cr.state
act_as_system_user do
- c.update_attributes!(state: Container::Complete)
+ c.update_attributes!(state: Container::Complete,
+ output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
+ log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
end
cr.reload
assert_equal "Final", cr.state
+ ['output', 'log'].each do |out_type|
+ pdh = Container.find_by_uuid(cr.container_uuid).send(out_type)
+ assert_equal(1, Collection.where(portable_data_hash: pdh,
+ owner_uuid: project.uuid).count,
+ "Container #{out_type} should be copied to #{project.uuid}")
+ end
end
test "Container makes container request, then is cancelled" do
state: Container::Complete,
exit_code: 0,
log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
- output: 'zzzzz-4zz18-znfnqtbbv4spc3w'
+ output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
}
c_older, _ = minimal_new(common_attrs)
completed_attrs = {
state: Container::Complete,
exit_code: 0,
- log: 'test',
+ log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
}
c_output1, _ = minimal_new(common_attrs)
set_user_from_auth :dispatch1
c_output1.update_attributes!({state: Container::Locked})
c_output1.update_attributes!({state: Container::Running})
- c_output1.update_attributes!(completed_attrs.merge({output: 'output 1'}))
+ c_output1.update_attributes!(completed_attrs.merge({output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'}))
c_output2.update_attributes!({state: Container::Locked})
c_output2.update_attributes!({state: Container::Running})
- c_output2.update_attributes!(completed_attrs.merge({output: 'output 2'}))
+ c_output2.update_attributes!(completed_attrs.merge({output: 'fa7aeb5140e2848d39b416daeef4ffc5+45'}))
reused = Container.find_reusable(common_attrs)
assert_nil reused
c_failed.update_attributes!({state: Container::Running})
c_failed.update_attributes!({state: Container::Complete,
exit_code: 42,
- log: "test",
- output: "test"})
+ log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
+ output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
c_running.update_attributes!({state: Container::Locked})
c_running.update_attributes!({state: Container::Running,
progress: 0.15})
c_completed.update_attributes!({state: Container::Running})
c_completed.update_attributes!({state: Container::Complete,
exit_code: 0,
- log: "ea10d51bcf88862dbcc36eb292017dfd+45",
- output: "zzzzz-4zz18-znfnqtbbv4spc3w"})
+ log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
+ output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
c_running.update_attributes!({state: Container::Locked})
c_running.update_attributes!({state: Container::Running,
progress: 0.15})
reused = Container.find_reusable(common_attrs)
assert_not_nil reused
- assert_equal reused.uuid, c_completed.uuid
+ assert_equal c_completed.uuid, reused.uuid
end
test "find_reusable method should select running over locked container" do
{active: "active/", admin: "admin/", system_user: ""}.
each_pair do |user_sym, name_prefix|
- %w(a aa a0 aA Aa AA A0).each do |name|
- test "'#{name_prefix}#{name}' is a valid name for #{user_sym} repo" do
+ test "valid names for #{user_sym} repo" do
+ %w(a aa a0 aA Aa AA A0).each do |name|
repo = new_repo(user_sym, name: name_prefix + name)
assert(repo.valid?)
end
refute(repo.valid?)
end
- "\\.-_/!@#$%^&*()[]{}".each_char do |bad_char|
- test "name containing #{bad_char.inspect} is invalid for #{user_sym}" do
+ test "name containing bad char is invalid for #{user_sym}" do
+ "\\.-_/!@#$%^&*()[]{}".each_char do |bad_char|
repo = new_repo(user_sym, name: "#{name_prefix}bad#{bad_char}reponame")
refute(repo.valid?)
end
api := httptest.NewServer(&apiStub)
defer api.Close()
- arv := arvadosclient.ArvadosClient{
+ arv := &arvadosclient.ArvadosClient{
Scheme: "http",
ApiServer: api.URL[7:],
ApiToken: "abc123",
api := httptest.NewServer(&apiStub)
defer api.Close()
- arv := arvadosclient.ArvadosClient{
+ arv := &arvadosclient.ArvadosClient{
Scheme: "http",
ApiServer: api.URL[7:],
ApiToken: "abc123",
type IArvadosClient interface {
Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error
Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
- Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error)
- Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) (err error)
+ Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
+ Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error
+ Discovery(key string) (interface{}, error)
}
// ErrCancelled is the error returned when the container is cancelled.
SigChan chan os.Signal
ArvMountExit chan error
finalState string
+ trashLifetime time.Duration
statLogger io.WriteCloser
statReporter *crunchstat.Reporter
err = runner.ArvClient.Create("collections",
arvadosclient.Dict{
"collection": arvadosclient.Dict{
+ "expires_at": time.Now().Add(runner.trashLifetime).Format(time.RFC3339),
+ "name": "output for " + runner.Container.UUID,
"manifest_text": manifestText}},
&response)
if err != nil {
return fmt.Errorf("While creating output collection: %v", err)
}
-
- runner.OutputPDH = new(string)
- *runner.OutputPDH = response.PortableDataHash
-
+ runner.OutputPDH = &response.PortableDataHash
return nil
}
+func (runner *ContainerRunner) loadDiscoveryVars() {
+ tl, err := runner.ArvClient.Discovery("defaultTrashLifetime")
+ if err != nil {
+ log.Fatalf("getting defaultTrashLifetime from discovery document: %s", err)
+ }
+ runner.trashLifetime = time.Duration(tl.(float64)) * time.Second
+}
+
func (runner *ContainerRunner) CleanupDirs() {
if runner.ArvMount != nil {
umount := exec.Command("fusermount", "-z", "-u", runner.ArvMountPoint)
err = runner.ArvClient.Create("collections",
arvadosclient.Dict{
"collection": arvadosclient.Dict{
+ "expires_at": time.Now().Add(runner.trashLifetime).Format(time.RFC3339),
"name": "logs for " + runner.Container.UUID,
"manifest_text": mt}},
&response)
if err != nil {
return fmt.Errorf("While creating log collection: %v", err)
}
-
runner.LogsPDH = &response.PortableDataHash
-
return nil
}
cr.Container.UUID = containerUUID
cr.CrunchLog = NewThrottledLogger(cr.NewLogWriter("crunch-run"))
cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0)
+ cr.loadDiscoveryVars()
return cr
}
api.Retries = 8
var kc *keepclient.KeepClient
- kc, err = keepclient.MakeKeepClient(&api)
+ kc, err = keepclient.MakeKeepClient(api)
if err != nil {
log.Fatalf("%s: %v", containerId, err)
}
return nil
}
+var discoveryMap = map[string]interface{}{"defaultTrashLifetime": float64(1209600)}
+
+func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
+ return discoveryMap[key], nil
+}
+
// CalledWith returns the parameters from the first API call whose
// parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
// "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
return nil
}
+func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
+ return discoveryMap[key], nil
+}
+
type KeepErrorTestClient struct{}
func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
// GetCollectionsParams params
type GetCollectionsParams struct {
- Client arvadosclient.ArvadosClient
+ Client *arvadosclient.ArvadosClient
Logger *logger.Logger
BatchSize int
}
api := httptest.NewServer(&apiStub)
defer api.Close()
- arv := arvadosclient.ArvadosClient{
+ arv := &arvadosclient.ArvadosClient{
Scheme: "http",
ApiServer: api.URL[7:],
ApiToken: "abc123",
var arvLogger *logger.Logger
-func singlerun(arv arvadosclient.ArvadosClient) error {
+func singlerun(arv *arvadosclient.ArvadosClient) error {
var err error
if isAdmin, err := util.UserIsAdmin(arv); err != nil {
return errors.New("Error verifying admin token: " + err.Error())
rlbss.Count)
}
- kc, err := keepclient.MakeKeepClient(&arv)
+ kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
return fmt.Errorf("Error setting up keep client %v", err.Error())
}
}
// BuildDataFetcher returns a data fetcher that fetches data from remote servers.
-func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
+func BuildDataFetcher(arv *arvadosclient.ArvadosClient) summary.DataFetcher {
return func(
arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
"time"
)
-var arv arvadosclient.ArvadosClient
+var arv *arvadosclient.ArvadosClient
var keepClient *keepclient.KeepClient
var keepServers []string
// keep client
keepClient = &keepclient.KeepClient{
- Arvados: &arv,
+ Arvados: arv,
Want_replicas: 2,
Client: &http.Client{},
}
// GetKeepServersParams struct
type GetKeepServersParams struct {
- Client arvadosclient.ArvadosClient
+ Client *arvadosclient.ArvadosClient
Logger *logger.Logger
Limit int
}
// GetServerContents of the keep server
func GetServerContents(arvLogger *logger.Logger,
keepServer ServerAddress,
- arv arvadosclient.ArvadosClient) (response ServerResponse) {
+ arv *arvadosclient.ArvadosClient) (response ServerResponse) {
err := GetServerStatus(arvLogger, keepServer, arv)
if err != nil {
// GetServerStatus get keep server status by invoking /status.json
func GetServerStatus(arvLogger *logger.Logger,
keepServer ServerAddress,
- arv arvadosclient.ArvadosClient) error {
+ arv *arvadosclient.ArvadosClient) error {
url := fmt.Sprintf("http://%s:%d/status.json",
keepServer.Host,
keepServer.Port)
// CreateIndexRequest to the keep server
func CreateIndexRequest(arvLogger *logger.Logger,
keepServer ServerAddress,
- arv arvadosclient.ArvadosClient) (req *http.Request, err error) {
+ arv *arvadosclient.ArvadosClient) (req *http.Request, err error) {
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
log.Println("About to fetch keep server contents from " + url)
tl := map[string]TrashList{
server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}}
- arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
- kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
+ arv := &arvadosclient.ArvadosClient{ApiToken: "abc123"}
+ kc := keepclient.KeepClient{Arvados: arv, Client: &http.Client{}}
kc.SetServiceRoots(map[string]string{"xxxx": server.URL},
map[string]string{"xxxx": server.URL},
map[string]string{})
tl := map[string]TrashList{
server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}}
- arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
- kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
+ arv := &arvadosclient.ArvadosClient{ApiToken: "abc123"}
+ kc := keepclient.KeepClient{Arvados: arv, Client: &http.Client{}}
kc.SetServiceRoots(map[string]string{"xxxx": server.URL},
map[string]string{"xxxx": server.URL},
map[string]string{})
api := httptest.NewServer(&apiStub)
defer api.Close()
- arv := arvadosclient.ArvadosClient{
+ arv := &arvadosclient.ArvadosClient{
Scheme: "http",
ApiServer: api.URL[7:],
ApiToken: "abc123",
Client: &http.Client{Transport: &http.Transport{}},
}
- kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
+ kc := keepclient.KeepClient{Arvados: arv, Client: &http.Client{}}
kc.SetServiceRoots(map[string]string{"xxxx": "http://example.com:23456"},
map[string]string{"xxxx": "http://example.com:23456"},
map[string]string{})
api := httptest.NewServer(&apiStub)
defer api.Close()
- arv := arvadosclient.ArvadosClient{
+ arv := &arvadosclient.ArvadosClient{
Scheme: "http",
ApiServer: api.URL[7:],
ApiToken: "abc123",
Client: &http.Client{Transport: &http.Transport{}},
}
- kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
+ kc := keepclient.KeepClient{Arvados: arv, Client: &http.Client{}}
kc.SetServiceRoots(map[string]string{"xxxx": ks.URL},
map[string]string{"xxxx": ks.URL},
map[string]string{})
arv.ApiToken = arvadostest.DataManagerToken
c.Assert(err, check.IsNil)
s.keepClient = &keepclient.KeepClient{
- Arvados: &arv,
+ Arvados: arv,
Client: &http.Client{},
}
c.Assert(s.keepClient.DiscoverKeepServers(), check.IsNil)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
arv.ApiToken = arvadostest.ActiveToken
- kc, err := keepclient.MakeKeepClient(&arv)
+ kc, err := keepclient.MakeKeepClient(arv)
c.Assert(err, check.Equals, nil)
loc, _, err := kc.PutB(testdata[:])
c.Assert(err, check.Equals, nil)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
arv.ApiToken = arvadostest.ActiveToken
- kc, err := keepclient.MakeKeepClient(&arv)
+ kc, err := keepclient.MakeKeepClient(arv)
c.Assert(err, check.Equals, nil)
kc.PutB([]byte("Hello world\n"))
kc.PutB([]byte("foo"))
if bogusClientToken {
arv.ApiToken = "bogus-token"
}
- kc := keepclient.New(&arv)
+ kc := keepclient.New(arv)
sr := map[string]string{
TestProxyUUID: "http://" + listener.Addr().String(),
}
c.Assert(err, Equals, nil)
// keepclient with no such keep server
- kc := keepclient.New(&arv)
+ kc := keepclient.New(arv)
locals := map[string]string{
TestProxyUUID: "http://localhost:12345",
}
// keep client
keepClient = &keepclient.KeepClient{
- Arvados: &arv,
+ Arvados: arv,
Want_replicas: 1,
Client: &http.Client{},
}
func RunTestPullWorker(c *C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- keepClient, err := keepclient.MakeKeepClient(&arv)
+ keepClient, err := keepclient.MakeKeepClient(arv)
c.Assert(err, Equals, nil)
pullq = NewWorkQueue()
projects / arvados.git / commitdiff