X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/351a497c96770f379acdfe58ebeff34262e1308b..3c0ef3fc0cdb3f56d90442cc389ce4c2c94e8831:/sdk/go/arvadosclient/arvadosclient.go diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go index 5ea2524aa6..18e1074bf6 100644 --- a/sdk/go/arvadosclient/arvadosclient.go +++ b/sdk/go/arvadosclient/arvadosclient.go @@ -12,20 +12,53 @@ import ( "net/http" "net/url" "os" + "regexp" "strings" + "time" ) -// Errors +type StringMatcher func(string) bool + +var UUIDMatch StringMatcher = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString +var PDHMatch StringMatcher = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString + var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST") var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN") +var ErrInvalidArgument = errors.New("Invalid argument") + +// A common failure mode is to reuse a keepalive connection that has been +// terminated (in a way that we can't detect) for being idle too long. +// POST and DELETE are not safe to retry automatically, so we minimize +// such failures by always using a new or recently active socket. +var MaxIdleConnectionDuration = 30 * time.Second + +// Indicates an error that was returned by the API server. +type APIServerError struct { + // Address of server returning error, of the form "host:port". + ServerAddress string + + // Components of server response. + HttpStatusCode int + HttpStatusMessage string -type ArvadosApiError struct { - error - HttpStatusCode int - HttpStatus string + // Additional error details from response body. + ErrorDetails []string } -func (e ArvadosApiError) Error() string { return e.error.Error() } +func (e APIServerError) Error() string { + if len(e.ErrorDetails) > 0 { + return fmt.Sprintf("arvados API server error: %s (%d: %s) returned by %s", + strings.Join(e.ErrorDetails, "; "), + e.HttpStatusCode, + e.HttpStatusMessage, + e.ServerAddress) + } else { + return fmt.Sprintf("arvados API server error: %d: %s returned by %s", + e.HttpStatusCode, + e.HttpStatusMessage, + e.ServerAddress) + } +} // Helper type so we don't have to write out 'map[string]interface{}' every time. type Dict map[string]interface{} @@ -47,16 +80,22 @@ type ArvadosClient struct { // If true, sets the X-External-Client header to indicate // the client is outside the cluster. External bool + + // Discovery document + DiscoveryDoc Dict + + lastClosedIdlesAt time.Time } -// Create a new KeepClient, initialized with standard Arvados environment +// Create a new ArvadosClient, initialized with standard Arvados environment // variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally) // ARVADOS_API_HOST_INSECURE. -func MakeArvadosClient() (kc ArvadosClient, err error) { - insecure := (os.Getenv("ARVADOS_API_HOST_INSECURE") == "true") - external := (os.Getenv("ARVADOS_EXTERNAL_CLIENT") == "true") +func MakeArvadosClient() (ac ArvadosClient, err error) { + var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$") + insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) + external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT")) - kc = ArvadosClient{ + ac = ArvadosClient{ ApiServer: os.Getenv("ARVADOS_API_HOST"), ApiToken: os.Getenv("ARVADOS_API_TOKEN"), ApiInsecure: insecure, @@ -64,38 +103,33 @@ func MakeArvadosClient() (kc ArvadosClient, err error) { TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}}, External: external} - if os.Getenv("ARVADOS_API_HOST") == "" { - return kc, MissingArvadosApiHost + if ac.ApiServer == "" { + return ac, MissingArvadosApiHost } - if os.Getenv("ARVADOS_API_TOKEN") == "" { - return kc, MissingArvadosApiToken + if ac.ApiToken == "" { + return ac, MissingArvadosApiToken } - return kc, err + ac.lastClosedIdlesAt = time.Now() + + return ac, err } -// Low-level access to a resource. -// -// method - HTTP method, one of GET, HEAD, PUT, POST or DELETE -// resource - the arvados resource to act on -// uuid - the uuid of the specific item to access (may be empty) -// action - sub-action to take on the resource or uuid (may be empty) -// parameters - method parameters -// -// return -// reader - the body reader, or nil if there was an error -// err - error accessing the resource, or nil if no error -func (this ArvadosClient) CallRaw(method string, resource string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) { +// CallRaw is the same as Call() but returns a Reader that reads the +// response body, instead of taking an output object. +func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) { var req *http.Request u := url.URL{ Scheme: "https", - Host: this.ApiServer} + Host: c.ApiServer} - u.Path = "/arvados/v1" + if resourceType != API_DISCOVERY_RESOURCE { + u.Path = "/arvados/v1" + } - if resource != "" { - u.Path = u.Path + "/" + resource + if resourceType != "" { + u.Path = u.Path + "/" + resourceType } if uuid != "" { u.Path = u.Path + "/" + uuid @@ -108,12 +142,11 @@ func (this ArvadosClient) CallRaw(method string, resource string, uuid string, a parameters = make(Dict) } - parameters["format"] = "json" - vals := make(url.Values) for k, v := range parameters { - m, err := json.Marshal(v) - if err == nil { + if s, ok := v.(string); ok { + vals.Set(k, s) + } else if m, err := json.Marshal(v); err == nil { vals.Set(k, string(m)) } } @@ -131,14 +164,23 @@ func (this ArvadosClient) CallRaw(method string, resource string, uuid string, a } // Add api token header - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", this.ApiToken)) - if this.External { + req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken)) + if c.External { req.Header.Add("X-External-Client", "1") } + // POST and DELETE are not safe to retry automatically, so we minimize + // such failures by always using a new or recently active socket + if method == "POST" || method == "DELETE" { + if time.Since(c.lastClosedIdlesAt) > MaxIdleConnectionDuration { + c.lastClosedIdlesAt = time.Now() + c.Client.Transport.(*http.Transport).CloseIdleConnections() + } + } + // Make the request var resp *http.Response - if resp, err = this.Client.Do(req); err != nil { + if resp, err = c.Client.Do(req); err != nil { return nil, err } @@ -147,45 +189,52 @@ func (this ArvadosClient) CallRaw(method string, resource string, uuid string, a } defer resp.Body.Close() - errorText := fmt.Sprintf("API response: %s", resp.Status) + return nil, newAPIServerError(c.ApiServer, resp) +} + +func newAPIServerError(ServerAddress string, resp *http.Response) APIServerError { + + ase := APIServerError{ + ServerAddress: ServerAddress, + HttpStatusCode: resp.StatusCode, + HttpStatusMessage: resp.Status} // If the response body has {"errors":["reason1","reason2"]} // then return those reasons. var errInfo = Dict{} if err := json.NewDecoder(resp.Body).Decode(&errInfo); err == nil { if errorList, ok := errInfo["errors"]; ok { - var errorStrings []string if errArray, ok := errorList.([]interface{}); ok { for _, errItem := range errArray { // We expect an array of strings here. // Non-strings will be passed along // JSON-encoded. if s, ok := errItem.(string); ok { - errorStrings = append(errorStrings, s) + ase.ErrorDetails = append(ase.ErrorDetails, s) } else if j, err := json.Marshal(errItem); err == nil { - errorStrings = append(errorStrings, string(j)) + ase.ErrorDetails = append(ase.ErrorDetails, string(j)) } } - errorText = strings.Join(errorStrings, "; ") } } } - return nil, ArvadosApiError{errors.New(errorText), resp.StatusCode, resp.Status} + return ase } -// Access to a resource. +// Call an API endpoint and parse the JSON response into an object. +// +// method - HTTP method: GET, HEAD, PUT, POST, PATCH or DELETE. +// resourceType - the type of arvados resource to act on (e.g., "collections", "pipeline_instances"). +// uuid - the uuid of the specific item to access. May be empty. +// action - API method name (e.g., "lock"). This is often empty if implied by method and uuid. +// parameters - method parameters. +// output - a map or annotated struct which is a legal target for encoding/json/Decoder. // -// method - HTTP method, one of GET, HEAD, PUT, POST or DELETE -// resource - the arvados resource to act on -// uuid - the uuid of the specific item to access (may be empty) -// action - sub-action to take on the resource or uuid (may be empty) -// parameters - method parameters -// output - a map or annotated struct which is a legal target for encoding/json/Decoder -// return -// err - error accessing the resource, or nil if no error -func (this ArvadosClient) Call(method string, resource string, uuid string, action string, parameters Dict, output interface{}) (err error) { - var reader io.ReadCloser - reader, err = this.CallRaw(method, resource, uuid, action, parameters) +// Returns a non-nil error if an error occurs making the API call, the +// API responds with a non-successful HTTP status, or an error occurs +// parsing the response body. +func (c ArvadosClient) Call(method string, resourceType string, uuid string, action string, parameters Dict, output interface{}) error { + reader, err := c.CallRaw(method, resourceType, uuid, action, parameters) if reader != nil { defer reader.Close() } @@ -202,48 +251,58 @@ func (this ArvadosClient) Call(method string, resource string, uuid string, acti return nil } -// Create a new instance of a resource. -// -// resource - the arvados resource on which to create an item -// parameters - method parameters -// output - a map or annotated struct which is a legal target for encoding/json/Decoder -// return -// err - error accessing the resource, or nil if no error -func (this ArvadosClient) Create(resource string, parameters Dict, output interface{}) (err error) { - return this.Call("POST", resource, "", "", parameters, output) +// Create a new resource. See Call for argument descriptions. +func (c ArvadosClient) Create(resourceType string, parameters Dict, output interface{}) error { + return c.Call("POST", resourceType, "", "", parameters, output) } -// Delete an instance of a resource. -// -// resource - the arvados resource on which to delete an item -// uuid - the item to delete -// parameters - method parameters -// output - a map or annotated struct which is a legal target for encoding/json/Decoder -// return -// err - error accessing the resource, or nil if no error -func (this ArvadosClient) Delete(resource string, uuid string, parameters Dict, output interface{}) (err error) { - return this.Call("DELETE", resource, uuid, "", parameters, output) +// Delete a resource. See Call for argument descriptions. +func (c ArvadosClient) Delete(resource string, uuid string, parameters Dict, output interface{}) (err error) { + return c.Call("DELETE", resource, uuid, "", parameters, output) } -// Update fields of an instance of a resource. -// -// resource - the arvados resource on which to update the item -// uuid - the item to update -// parameters - method parameters -// output - a map or annotated struct which is a legal target for encoding/json/Decoder -// return -// err - error accessing the resource, or nil if no error -func (this ArvadosClient) Update(resource string, uuid string, parameters Dict, output interface{}) (err error) { - return this.Call("PUT", resource, uuid, "", parameters, output) +// Modify attributes of a resource. See Call for argument descriptions. +func (c ArvadosClient) Update(resourceType string, uuid string, parameters Dict, output interface{}) (err error) { + return c.Call("PUT", resourceType, uuid, "", parameters, output) } -// List the instances of a resource -// -// resource - the arvados resource on which to list -// parameters - method parameters -// output - a map or annotated struct which is a legal target for encoding/json/Decoder -// return -// err - error accessing the resource, or nil if no error -func (this ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) { - return this.Call("GET", resource, "", "", parameters, output) +// Get a resource. See Call for argument descriptions. +func (c ArvadosClient) Get(resourceType string, uuid string, parameters Dict, output interface{}) (err error) { + if !UUIDMatch(uuid) && !(resourceType == "collections" && PDHMatch(uuid)) { + // No object has uuid == "": there is no need to make + // an API call. Furthermore, the HTTP request for such + // an API call would be "GET /arvados/v1/type/", which + // is liable to be misinterpreted as the List API. + return ErrInvalidArgument + } + return c.Call("GET", resourceType, uuid, "", parameters, output) +} + +// List resources of a given type. See Call for argument descriptions. +func (c ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) { + return c.Call("GET", resource, "", "", parameters, output) +} + +const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest" + +// Discovery returns the value of the given parameter in the discovery +// document. Returns a non-nil error if the discovery document cannot +// be retrieved/decoded. Returns ErrInvalidArgument if the requested +// parameter is not found in the discovery document. +func (c *ArvadosClient) Discovery(parameter string) (value interface{}, err error) { + if len(c.DiscoveryDoc) == 0 { + c.DiscoveryDoc = make(Dict) + err = c.Call("GET", API_DISCOVERY_RESOURCE, "", "", nil, &c.DiscoveryDoc) + if err != nil { + return nil, err + } + } + + var found bool + value, found = c.DiscoveryDoc[parameter] + if found { + return value, nil + } else { + return value, ErrInvalidArgument + } }