From d161e2280a03b4a1c9675f5ed1946310a9942acd Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 10 Jun 2014 14:56:01 -0400 Subject: [PATCH] Merge branch '2826-simple-go-sdk' closes #2826 --- .../src/arvados.org/keepclient/keepclient.go | 42 +--- .../arvados.org/keepclient/keepclient_test.go | 76 +++--- sdk/go/src/arvados.org/keepclient/support.go | 51 +--- sdk/go/src/arvados.org/sdk/sdk.go | 232 ++++++++++++++++++ sdk/go/src/arvados.org/sdk/sdk_test.go | 92 +++++++ .../src/arvados.org/keepproxy/keepproxy.go | 32 +-- .../arvados.org/keepproxy/keepproxy_test.go | 9 +- 7 files changed, 399 insertions(+), 135 deletions(-) create mode 100644 sdk/go/src/arvados.org/sdk/sdk.go create mode 100644 sdk/go/src/arvados.org/sdk/sdk_test.go diff --git a/sdk/go/src/arvados.org/keepclient/keepclient.go b/sdk/go/src/arvados.org/keepclient/keepclient.go index ee91d6f1a1..d43a2150f3 100644 --- a/sdk/go/src/arvados.org/keepclient/keepclient.go +++ b/sdk/go/src/arvados.org/keepclient/keepclient.go @@ -2,16 +2,15 @@ package keepclient import ( + "arvados.org/sdk" "arvados.org/streamer" "crypto/md5" - "crypto/tls" "errors" "fmt" "io" "io/ioutil" "log" "net/http" - "os" "regexp" "sort" "strings" @@ -34,41 +33,22 @@ const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored" // Information about Arvados and Keep servers. type KeepClient struct { - ApiServer string - ApiToken string - ApiInsecure bool + Arvados *sdk.ArvadosClient Want_replicas int - Client *http.Client Using_proxy bool - External bool service_roots *[]string lock sync.Mutex + Client *http.Client } -// Create a new KeepClient, initialized with standard Arvados environment -// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally) -// ARVADOS_API_HOST_INSECURE. This will contact the API server to discover -// Keep servers. -func MakeKeepClient() (kc KeepClient, err error) { - insecure := (os.Getenv("ARVADOS_API_HOST_INSECURE") == "true") - external := (os.Getenv("ARVADOS_EXTERNAL_CLIENT") == "true") - +// Create a new KeepClient. This will contact the API server to discover Keep +// servers. +func MakeKeepClient(arv *sdk.ArvadosClient) (kc KeepClient, err error) { kc = KeepClient{ - ApiServer: os.Getenv("ARVADOS_API_HOST"), - ApiToken: os.Getenv("ARVADOS_API_TOKEN"), - ApiInsecure: insecure, + Arvados: arv, Want_replicas: 2, - Client: &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}}, - Using_proxy: false, - External: external} - - if os.Getenv("ARVADOS_API_HOST") == "" { - return kc, MissingArvadosApiHost - } - if os.Getenv("ARVADOS_API_TOKEN") == "" { - return kc, MissingArvadosApiToken - } + Using_proxy: false, + Client: &http.Client{Transport: &http.Transport{}}} err = (&kc).DiscoverKeepServers() @@ -169,7 +149,7 @@ func (this KeepClient) AuthorizedGet(hash string, continue } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken)) + req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken)) var resp *http.Response if resp, err = this.Client.Do(req); err != nil { @@ -211,7 +191,7 @@ func (this KeepClient) AuthorizedAsk(hash string, signature string, continue } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken)) + req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken)) var resp *http.Response if resp, err = this.Client.Do(req); err != nil { diff --git a/sdk/go/src/arvados.org/keepclient/keepclient_test.go b/sdk/go/src/arvados.org/keepclient/keepclient_test.go index 753a0ace28..5327fb507e 100644 --- a/sdk/go/src/arvados.org/keepclient/keepclient_test.go +++ b/sdk/go/src/arvados.org/keepclient/keepclient_test.go @@ -1,6 +1,7 @@ package keepclient import ( + "arvados.org/sdk" "arvados.org/streamer" "crypto/md5" "flag" @@ -62,20 +63,12 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) { func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) { os.Setenv("ARVADOS_API_HOST", "localhost:3001") os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") - os.Setenv("ARVADOS_API_HOST_INSECURE", "") - - kc, err := MakeKeepClient() - c.Check(kc.ApiServer, Equals, "localhost:3001") - c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") - c.Check(kc.ApiInsecure, Equals, false) - os.Setenv("ARVADOS_API_HOST_INSECURE", "true") - kc, err = MakeKeepClient() - c.Check(kc.ApiServer, Equals, "localhost:3001") - c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") - c.Check(kc.ApiInsecure, Equals, true) - c.Check(kc.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true) + arv, err := sdk.MakeArvadosClient() + c.Assert(err, Equals, nil) + + kc, err := MakeKeepClient(&arv) c.Assert(err, Equals, nil) c.Check(len(kc.ServiceRoots()), Equals, 2) @@ -133,8 +126,10 @@ func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string, listener, url := RunBogusKeepServer(st, 2990) defer listener.Close() - kc, _ := MakeKeepClient() - kc.ApiToken = "abc123" + arv, _ := sdk.MakeArvadosClient() + arv.ApiToken = "abc123" + + kc, _ := MakeKeepClient(&arv) reader, writer := io.Pipe() upload_status := make(chan uploadStatus) @@ -265,10 +260,11 @@ func (s *StandaloneSuite) TestPutB(c *C) { "foo", make(chan string, 2)} - kc, _ := MakeKeepClient() + arv, _ := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 - kc.ApiToken = "abc123" + arv.ApiToken = "abc123" service_roots := make([]string, 5) ks := RunSomeFakeKeepServers(st, 5, 2990) @@ -306,10 +302,11 @@ func (s *StandaloneSuite) TestPutHR(c *C) { "foo", make(chan string, 2)} - kc, _ := MakeKeepClient() + arv, _ := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 - kc.ApiToken = "abc123" + arv.ApiToken = "abc123" service_roots := make([]string, 5) ks := RunSomeFakeKeepServers(st, 5, 2990) @@ -359,10 +356,11 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) { fh := FailHandler{ make(chan string, 1)} - kc, _ := MakeKeepClient() + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 - kc.ApiToken = "abc123" + arv.ApiToken = "abc123" service_roots := make([]string, 5) ks1 := RunSomeFakeKeepServers(st, 4, 2990) @@ -407,10 +405,11 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { fh := FailHandler{ make(chan string, 4)} - kc, _ := MakeKeepClient() + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 - kc.ApiToken = "abc123" + arv.ApiToken = "abc123" service_roots := make([]string, 5) ks1 := RunSomeFakeKeepServers(st, 1, 2990) @@ -466,8 +465,9 @@ func (s *StandaloneSuite) TestGet(c *C) { listener, url := RunBogusKeepServer(st, 2990) defer listener.Close() - kc, _ := MakeKeepClient() - kc.ApiToken = "abc123" + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" kc.SetServiceRoots([]string{url}) r, n, url2, err := kc.Get(hash) @@ -491,8 +491,9 @@ func (s *StandaloneSuite) TestGetFail(c *C) { listener, url := RunBogusKeepServer(st, 2990) defer listener.Close() - kc, _ := MakeKeepClient() - kc.ApiToken = "abc123" + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" kc.SetServiceRoots([]string{url}) r, n, url2, err := kc.Get(hash) @@ -520,8 +521,9 @@ func (s *StandaloneSuite) TestChecksum(c *C) { listener, url := RunBogusKeepServer(st, 2990) defer listener.Close() - kc, _ := MakeKeepClient() - kc.ApiToken = "abc123" + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" kc.SetServiceRoots([]string{url}) r, n, _, err := kc.Get(barhash) @@ -552,8 +554,9 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { "abc123", []byte("foo")} - kc, _ := MakeKeepClient() - kc.ApiToken = "abc123" + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) + arv.ApiToken = "abc123" service_roots := make([]string, 5) ks1 := RunSomeFakeKeepServers(st, 1, 2990) @@ -586,7 +589,8 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) { os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") os.Setenv("ARVADOS_API_HOST_INSECURE", "true") - kc, err := MakeKeepClient() + arv, err := sdk.MakeArvadosClient() + kc, err := MakeKeepClient(&arv) c.Assert(err, Equals, nil) hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) @@ -634,11 +638,12 @@ func (s *StandaloneSuite) TestPutProxy(c *C) { st := StubProxyHandler{make(chan string, 1)} - kc, _ := MakeKeepClient() + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 2 kc.Using_proxy = true - kc.ApiToken = "abc123" + arv.ApiToken = "abc123" service_roots := make([]string, 1) ks1 := RunSomeFakeKeepServers(st, 1, 2990) @@ -664,11 +669,12 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) { st := StubProxyHandler{make(chan string, 1)} - kc, _ := MakeKeepClient() + arv, err := sdk.MakeArvadosClient() + kc, _ := MakeKeepClient(&arv) kc.Want_replicas = 3 kc.Using_proxy = true - kc.ApiToken = "abc123" + arv.ApiToken = "abc123" service_roots := make([]string, 1) ks1 := RunSomeFakeKeepServers(st, 1, 2990) diff --git a/sdk/go/src/arvados.org/keepclient/support.go b/sdk/go/src/arvados.org/keepclient/support.go index 5aeec125a1..f3e47f92f5 100644 --- a/sdk/go/src/arvados.org/keepclient/support.go +++ b/sdk/go/src/arvados.org/keepclient/support.go @@ -3,7 +3,6 @@ package keepclient import ( "arvados.org/streamer" - "encoding/json" "errors" "fmt" "io" @@ -29,53 +28,17 @@ func (this *KeepClient) DiscoverKeepServers() error { return nil } - // Construct request of keep disk list - var req *http.Request - var err error - - if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_services/accessible?format=json", this.ApiServer), nil); err != nil { - return err - } - - // Add api token header - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken)) - if this.External { - req.Header.Add("X-External-Client", "1") + type svcList struct { + Items []keepDisk `json:"items"` } + var m svcList - // Make the request - var resp *http.Response - if resp, err = this.Client.Do(req); err != nil { - return err - } + err := this.Arvados.Call("GET", "keep_services", "", "accessible", nil, &m) - if resp.StatusCode != http.StatusOK { - // fall back on keep disks - if req, err = http.NewRequest("GET", fmt.Sprintf("https://%s/arvados/v1/keep_disks", this.ApiServer), nil); err != nil { - return err - } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken)) - if resp, err = this.Client.Do(req); err != nil { + if err != nil { + if err := this.Arvados.List("keep_disks", nil, &m); err != nil { return err } - if resp.StatusCode != http.StatusOK { - return errors.New(resp.Status) - } - } - - // 'defer' is a stack, so it will drain the Body before closing it. - defer resp.Body.Close() - defer io.Copy(ioutil.Discard, resp.Body) - - type svcList struct { - Items []keepDisk `json:"items"` - } - - // Decode json reply - dec := json.NewDecoder(resp.Body) - var m svcList - if err := dec.Decode(&m); err != nil { - return err } listed := make(map[string]bool) @@ -181,7 +144,7 @@ func (this KeepClient) uploadToKeepServer(host string, hash string, body io.Read req.ContentLength = expectedLength } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken)) + req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.Arvados.ApiToken)) req.Header.Add("Content-Type", "application/octet-stream") if this.Using_proxy { diff --git a/sdk/go/src/arvados.org/sdk/sdk.go b/sdk/go/src/arvados.org/sdk/sdk.go new file mode 100644 index 0000000000..9f1880fab1 --- /dev/null +++ b/sdk/go/src/arvados.org/sdk/sdk.go @@ -0,0 +1,232 @@ +/* Simple Arvados Go SDK for communicating with API server. */ + +package sdk + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" +) + +// Errors +var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST") +var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN") +var ArvadosErrorForbidden = errors.New("Forbidden") +var ArvadosErrorNotFound = errors.New("Not found") +var ArvadosErrorBadRequest = errors.New("Bad request") +var ArvadosErrorServerError = errors.New("Server error") + +// Helper type so we don't have to write out 'map[string]interface{}' every time. +type Dict map[string]interface{} + +// Information about how to contact the Arvados server +type ArvadosClient struct { + // Arvados API server, form "host:port" + ApiServer string + + // Arvados API token for authentication + ApiToken string + + // Whether to require a valid SSL certificate or not + ApiInsecure bool + + // Client object shared by client requests. Supports HTTP KeepAlive. + Client *http.Client + + // If true, sets the X-External-Client header to indicate + // the client is outside the cluster. + External bool +} + +// Create a new KeepClient, initialized with standard Arvados environment +// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally) +// ARVADOS_API_HOST_INSECURE. +func MakeArvadosClient() (kc ArvadosClient, err error) { + insecure := (os.Getenv("ARVADOS_API_HOST_INSECURE") == "true") + external := (os.Getenv("ARVADOS_EXTERNAL_CLIENT") == "true") + + kc = ArvadosClient{ + ApiServer: os.Getenv("ARVADOS_API_HOST"), + ApiToken: os.Getenv("ARVADOS_API_TOKEN"), + ApiInsecure: insecure, + Client: &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}}, + External: external} + + if os.Getenv("ARVADOS_API_HOST") == "" { + return kc, MissingArvadosApiHost + } + if os.Getenv("ARVADOS_API_TOKEN") == "" { + return kc, MissingArvadosApiToken + } + + return kc, err +} + +// Low-level access to a resource. +// +// method - HTTP method, one of GET, HEAD, PUT, POST or DELETE +// resource - the arvados resource to act on +// uuid - the uuid of the specific item to access (may be empty) +// action - sub-action to take on the resource or uuid (may be empty) +// parameters - method parameters +// +// return +// reader - the body reader, or nil if there was an error +// err - error accessing the resource, or nil if no error +func (this ArvadosClient) CallRaw(method string, resource string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) { + var req *http.Request + + u := url.URL{ + Scheme: "https", + Host: this.ApiServer} + + u.Path = "/arvados/v1" + + if resource != "" { + u.Path = u.Path + "/" + resource + } + if uuid != "" { + u.Path = u.Path + "/" + uuid + } + if action != "" { + u.Path = u.Path + "/" + action + } + + if parameters == nil { + parameters = make(Dict) + } + + parameters["format"] = "json" + + vals := make(url.Values) + for k, v := range parameters { + m, err := json.Marshal(v) + if err == nil { + vals.Set(k, string(m)) + } + } + + if method == "GET" || method == "HEAD" { + u.RawQuery = vals.Encode() + if req, err = http.NewRequest(method, u.String(), nil); err != nil { + return nil, err + } + } else { + if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil { + return nil, err + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + } + + // Add api token header + req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken)) + if this.External { + req.Header.Add("X-External-Client", "1") + } + + // Make the request + var resp *http.Response + if resp, err = this.Client.Do(req); err != nil { + return nil, err + } + + switch resp.StatusCode { + case http.StatusOK: + return resp.Body, nil + case http.StatusForbidden: + resp.Body.Close() + return nil, ArvadosErrorForbidden + case http.StatusNotFound: + resp.Body.Close() + return nil, ArvadosErrorNotFound + default: + resp.Body.Close() + if resp.StatusCode >= 400 && resp.StatusCode <= 499 { + return nil, ArvadosErrorBadRequest + } else { + return nil, ArvadosErrorServerError + } + } +} + +// Access to a resource. +// +// method - HTTP method, one of GET, HEAD, PUT, POST or DELETE +// resource - the arvados resource to act on +// uuid - the uuid of the specific item to access (may be empty) +// action - sub-action to take on the resource or uuid (may be empty) +// parameters - method parameters +// output - a map or annotated struct which is a legal target for encoding/json/Decoder +// return +// err - error accessing the resource, or nil if no error +func (this ArvadosClient) Call(method string, resource string, uuid string, action string, parameters Dict, output interface{}) (err error) { + var reader io.ReadCloser + reader, err = this.CallRaw(method, resource, uuid, action, parameters) + if reader != nil { + defer reader.Close() + } + if err != nil { + return err + } + + if output != nil { + dec := json.NewDecoder(reader) + if err = dec.Decode(output); err != nil { + return err + } + } + return nil +} + +// Create a new instance of a resource. +// +// resource - the arvados resource on which to create an item +// parameters - method parameters +// output - a map or annotated struct which is a legal target for encoding/json/Decoder +// return +// err - error accessing the resource, or nil if no error +func (this ArvadosClient) Create(resource string, parameters Dict, output interface{}) (err error) { + return this.Call("POST", resource, "", "", parameters, output) +} + +// Delete an instance of a resource. +// +// resource - the arvados resource on which to delete an item +// uuid - the item to delete +// parameters - method parameters +// output - a map or annotated struct which is a legal target for encoding/json/Decoder +// return +// err - error accessing the resource, or nil if no error +func (this ArvadosClient) Delete(resource string, uuid string, parameters Dict, output interface{}) (err error) { + return this.Call("DELETE", resource, uuid, "", parameters, output) +} + +// Update fields of an instance of a resource. +// +// resource - the arvados resource on which to update the item +// uuid - the item to update +// parameters - method parameters +// output - a map or annotated struct which is a legal target for encoding/json/Decoder +// return +// err - error accessing the resource, or nil if no error +func (this ArvadosClient) Update(resource string, uuid string, parameters Dict, output interface{}) (err error) { + return this.Call("PUT", resource, uuid, "", parameters, output) +} + +// List the instances of a resource +// +// resource - the arvados resource on which to list +// parameters - method parameters +// output - a map or annotated struct which is a legal target for encoding/json/Decoder +// return +// err - error accessing the resource, or nil if no error +func (this ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) { + return this.Call("GET", resource, "", "", parameters, output) +} diff --git a/sdk/go/src/arvados.org/sdk/sdk_test.go b/sdk/go/src/arvados.org/sdk/sdk_test.go new file mode 100644 index 0000000000..8af848f8d6 --- /dev/null +++ b/sdk/go/src/arvados.org/sdk/sdk_test.go @@ -0,0 +1,92 @@ +package sdk + +import ( + "fmt" + . "gopkg.in/check.v1" + "net/http" + "os" + "os/exec" + "strings" + "testing" +) + +// Gocheck boilerplate +func Test(t *testing.T) { + TestingT(t) +} + +var _ = Suite(&ServerRequiredSuite{}) + +// Tests that require the Keep server running +type ServerRequiredSuite struct{} + +func pythonDir() string { + gopath := os.Getenv("GOPATH") + return fmt.Sprintf("%s/../python/tests", strings.Split(gopath, ":")[0]) +} + +func (s *ServerRequiredSuite) SetUpSuite(c *C) { + os.Chdir(pythonDir()) + if err := exec.Command("python", "run_test_server.py", "start").Run(); err != nil { + panic("'python run_test_server.py start' returned error") + } + if err := exec.Command("python", "run_test_server.py", "start_keep").Run(); err != nil { + panic("'python run_test_server.py start_keep' returned error") + } +} + +func (s *ServerRequiredSuite) TestMakeArvadosClient(c *C) { + os.Setenv("ARVADOS_API_HOST", "localhost:3001") + os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") + os.Setenv("ARVADOS_API_HOST_INSECURE", "") + + kc, err := MakeArvadosClient() + c.Check(kc.ApiServer, Equals, "localhost:3001") + c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") + c.Check(kc.ApiInsecure, Equals, false) + + os.Setenv("ARVADOS_API_HOST_INSECURE", "true") + + kc, err = MakeArvadosClient() + c.Check(kc.ApiServer, Equals, "localhost:3001") + c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") + c.Check(kc.ApiInsecure, Equals, true) + c.Check(kc.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true) + + c.Assert(err, Equals, nil) +} + +func (s *ServerRequiredSuite) TestCreatePipelineTemplate(c *C) { + os.Setenv("ARVADOS_API_HOST", "localhost:3001") + os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") + os.Setenv("ARVADOS_API_HOST_INSECURE", "true") + + arv, err := MakeArvadosClient() + + getback := make(Dict) + err = arv.Create("pipeline_templates", + Dict{"pipeline_template": Dict{ + "name": "tmp", + "components": Dict{ + "c1": map[string]string{"script": "script1"}, + "c2": map[string]string{"script": "script2"}}}}, + &getback) + c.Assert(err, Equals, nil) + c.Assert(getback["name"], Equals, "tmp") + c.Assert(getback["components"].(map[string]interface{})["c2"].(map[string]interface{})["script"], Equals, "script2") + + uuid := getback["uuid"].(string) + getback = make(Dict) + err = arv.Update("pipeline_templates", uuid, + Dict{ + "pipeline_template": Dict{"name": "tmp2"}}, + &getback) + c.Assert(err, Equals, nil) + c.Assert(getback["name"], Equals, "tmp2") + + c.Assert(getback["uuid"].(string), Equals, uuid) + getback = make(Dict) + err = arv.Delete("pipeline_templates", uuid, nil, &getback) + c.Assert(err, Equals, nil) + c.Assert(getback["name"], Equals, "tmp2") +} diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy.go b/services/keep/src/arvados.org/keepproxy/keepproxy.go index 42f5a78231..56de1e16d7 100644 --- a/services/keep/src/arvados.org/keepproxy/keepproxy.go +++ b/services/keep/src/arvados.org/keepproxy/keepproxy.go @@ -2,6 +2,7 @@ package main import ( "arvados.org/keepclient" + "arvados.org/sdk" "flag" "fmt" "github.com/gorilla/mux" @@ -67,7 +68,12 @@ func main() { flagset.Parse(os.Args[1:]) - kc, err := keepclient.MakeKeepClient() + arv, err := sdk.MakeArvadosClient() + if err != nil { + log.Fatalf("Error setting up arvados client %s", err.Error()) + } + + kc, err := keepclient.MakeKeepClient(&arv) if err != nil { log.Fatalf("Error setting up keep client %s", err.Error()) } @@ -205,31 +211,13 @@ func CheckAuthorizationHeader(kc keepclient.KeepClient, cache *ApiTokenCache, re return true } - var usersreq *http.Request - - if usersreq, err = http.NewRequest("HEAD", fmt.Sprintf("https://%s/arvados/v1/users/current", kc.ApiServer), nil); err != nil { - // Can't construct the request + arv := *kc.Arvados + arv.ApiToken = tok + if err := arv.Call("HEAD", "users", "", "current", nil, nil); err != nil { log.Printf("%s: CheckAuthorizationHeader error: %v", GetRemoteAddress(req), err) return false } - // Add api token header - usersreq.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", tok)) - - // Actually make the request - var resp *http.Response - if resp, err = kc.Client.Do(usersreq); err != nil { - // Something else failed - log.Printf("%s: CheckAuthorizationHeader error connecting to API server: %v", GetRemoteAddress(req), err.Error()) - return false - } - - if resp.StatusCode != http.StatusOK { - // Bad status - log.Printf("%s: CheckAuthorizationHeader API server responded: %v", GetRemoteAddress(req), resp.Status) - return false - } - // Success! Update cache cache.RememberToken(tok) diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go index 47c33b4e1b..4bf347890e 100644 --- a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go +++ b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go @@ -2,6 +2,7 @@ package main import ( "arvados.org/keepclient" + "arvados.org/sdk" "crypto/md5" "crypto/tls" "fmt" @@ -108,7 +109,8 @@ func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port)) os.Setenv("ARVADOS_API_TOKEN", token) - kc, err := keepclient.MakeKeepClient() + arv, err := sdk.MakeArvadosClient() + kc, err := keepclient.MakeKeepClient(&arv) c.Check(kc.Using_proxy, Equals, true) c.Check(len(kc.ServiceRoots()), Equals, 1) c.Check(kc.ServiceRoots()[0], Equals, fmt.Sprintf("http://localhost:%v", port)) @@ -129,8 +131,9 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { setupProxyService() os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true") - kc, err := keepclient.MakeKeepClient() - c.Check(kc.External, Equals, true) + arv, err := sdk.MakeArvadosClient() + kc, err := keepclient.MakeKeepClient(&arv) + c.Check(kc.Arvados.External, Equals, true) c.Check(kc.Using_proxy, Equals, true) c.Check(len(kc.ServiceRoots()), Equals, 1) c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:29950") -- 2.30.2