+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
/* Simple Arvados Go SDK for communicating with API server. */
package arvadosclient
"fmt"
"io"
"io/ioutil"
+ "log"
"net/http"
"net/url"
"os"
- "regexp"
"strings"
+ "sync"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
)
type StringMatcher func(string) bool
-var UUIDMatch StringMatcher = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString
-var PDHMatch StringMatcher = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString
+var UUIDMatch StringMatcher = arvados.UUIDMatch
+var PDHMatch StringMatcher = arvados.PDHMatch
var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
var RetryDelay = 2 * time.Second
-// Indicates an error that was returned by the API server.
+var (
+ defaultInsecureHTTPClient *http.Client
+ defaultSecureHTTPClient *http.Client
+ defaultHTTPClientMtx sync.Mutex
+)
+
+// APIServerError contains an error that was returned by the API server.
type APIServerError struct {
// Address of server returning error, of the form "host:port".
ServerAddress string
e.HttpStatusCode,
e.HttpStatusMessage,
e.ServerAddress)
- } else {
- return fmt.Sprintf("arvados API server error: %d: %s returned by %s",
- e.HttpStatusCode,
- e.HttpStatusMessage,
- e.ServerAddress)
}
+ return fmt.Sprintf("arvados API server error: %d: %s returned by %s",
+ e.HttpStatusCode,
+ e.HttpStatusMessage,
+ e.ServerAddress)
}
-// Helper type so we don't have to write out 'map[string]interface{}' every time.
+// StringBool tests whether s is suggestive of true. It returns true
+// if s is a mixed/uppoer/lower-case variant of "1", "yes", or "true".
+func StringBool(s string) bool {
+ s = strings.ToLower(s)
+ return s == "1" || s == "yes" || s == "true"
+}
+
+// Dict is a helper type so we don't have to write out 'map[string]interface{}' every time.
type Dict map[string]interface{}
-// Information about how to contact the Arvados server
+// ArvadosClient contains information about how to contact the Arvados server
type ArvadosClient struct {
// https
Scheme string
// Client object shared by client requests. Supports HTTP KeepAlive.
Client *http.Client
- // If true, sets the X-External-Client header to indicate
- // the client is outside the cluster.
- External bool
-
// Base URIs of Keep services, e.g., {"https://host1:8443",
// "https://host2:8443"}. If this is nil, Keep clients will
// use the arvados.v1.keep_services.accessible API to discover
// Number of retries
Retries int
+
+ // X-Request-Id for outgoing requests
+ RequestID string
}
var CertFiles = []string{
"/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL
}
-// MakeTLSConfig sets up TLS configuration for communicating with Arvados and Keep services.
+// MakeTLSConfig sets up TLS configuration for communicating with
+// Arvados and Keep services.
func MakeTLSConfig(insecure bool) *tls.Config {
tlsconfig := tls.Config{InsecureSkipVerify: insecure}
if !insecure {
- // Look for /etc/arvados/ca-certificates.crt in addition to normal system certs.
+ // Use the first entry in CertFiles that we can read
+ // certificates from. If none of those work out, use
+ // the Go defaults.
certs := x509.NewCertPool()
for _, file := range CertFiles {
data, err := ioutil.ReadFile(file)
- if err == nil {
- success := certs.AppendCertsFromPEM(data)
- if !success {
- fmt.Printf("Unable to load any certificates from %v", file)
- } else {
- tlsconfig.RootCAs = certs
- break
+ if err != nil {
+ if !os.IsNotExist(err) {
+ log.Printf("proceeding without loading cert file %q: %s", file, err)
}
+ continue
}
+ if !certs.AppendCertsFromPEM(data) {
+ log.Printf("unable to load any certificates from %v", file)
+ continue
+ }
+ tlsconfig.RootCAs = certs
+ break
}
- // Will use system default CA roots instead.
}
return &tlsconfig
// fields from configuration files but still need to use the
// arvadosclient.ArvadosClient package.
func New(c *arvados.Client) (*ArvadosClient, error) {
+ hc := c.Client
+ if hc == nil {
+ hc = &http.Client{
+ Timeout: 5 * time.Minute,
+ Transport: &http.Transport{
+ TLSClientConfig: MakeTLSConfig(c.Insecure)},
+ }
+ }
ac := &ArvadosClient{
- Scheme: "https",
- ApiServer: c.APIHost,
- ApiToken: c.AuthToken,
- ApiInsecure: c.Insecure,
- Client: &http.Client{Transport: &http.Transport{
- TLSClientConfig: MakeTLSConfig(c.Insecure)}},
- External: false,
+ Scheme: "https",
+ ApiServer: c.APIHost,
+ ApiToken: c.AuthToken,
+ ApiInsecure: c.Insecure,
+ Client: hc,
Retries: 2,
+ KeepServiceURIs: c.KeepServiceURIs,
lastClosedIdlesAt: time.Now(),
}
// MakeArvadosClient creates a new ArvadosClient using the standard
// environment variables ARVADOS_API_HOST, ARVADOS_API_TOKEN,
-// ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT, and
-// ARVADOS_KEEP_SERVICES.
-func MakeArvadosClient() (ac *ArvadosClient, err error) {
- var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
- insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
- external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
-
- ac = &ArvadosClient{
- Scheme: "https",
- ApiServer: os.Getenv("ARVADOS_API_HOST"),
- ApiToken: os.Getenv("ARVADOS_API_TOKEN"),
- ApiInsecure: insecure,
- Client: &http.Client{Transport: &http.Transport{
- TLSClientConfig: MakeTLSConfig(insecure)}},
- External: external,
- Retries: 2}
-
- for _, s := range strings.Split(os.Getenv("ARVADOS_KEEP_SERVICES"), " ") {
- if s == "" {
- continue
- }
- if u, err := url.Parse(s); err != nil {
- return ac, fmt.Errorf("ARVADOS_KEEP_SERVICES: %q: %s", s, err)
- } else if !u.IsAbs() {
- return ac, fmt.Errorf("ARVADOS_KEEP_SERVICES: %q: not an absolute URI", s)
- }
- ac.KeepServiceURIs = append(ac.KeepServiceURIs, s)
- }
-
- if ac.ApiServer == "" {
- return ac, MissingArvadosApiHost
- }
- if ac.ApiToken == "" {
- return ac, MissingArvadosApiToken
- }
-
- ac.lastClosedIdlesAt = time.Now()
-
- return ac, err
+// ARVADOS_API_HOST_INSECURE, and ARVADOS_KEEP_SERVICES.
+func MakeArvadosClient() (*ArvadosClient, error) {
+ return New(arvados.NewClientFromEnv())
}
// CallRaw is the same as Call() but returns a Reader that reads the
if scheme == "" {
scheme = "https"
}
+ if c.ApiServer == "" {
+ return nil, fmt.Errorf("Arvados client is not configured (target API host is not set). Maybe env var ARVADOS_API_HOST should be set first?")
+ }
u := url.URL{
Scheme: scheme,
Host: c.ApiServer}
- if resourceType != API_DISCOVERY_RESOURCE {
+ if resourceType != ApiDiscoveryResource {
u.Path = "/arvados/v1"
}
vals.Set(k, string(m))
}
}
-
- retryable := false
- switch method {
- case "GET", "HEAD", "PUT", "OPTIONS", "DELETE":
- retryable = true
- }
-
- // Non-retryable methods such as POST are not safe to retry automatically,
- // so we minimize such failures by always using a new or recently active socket
- if !retryable {
- if time.Since(c.lastClosedIdlesAt) > MaxIdleConnectionDuration {
- c.lastClosedIdlesAt = time.Now()
- c.Client.Transport.(*http.Transport).CloseIdleConnections()
- }
- }
-
- // Make the request
var req *http.Request
- var resp *http.Response
-
- for attempt := 0; attempt <= c.Retries; attempt++ {
- if method == "GET" || method == "HEAD" {
- u.RawQuery = vals.Encode()
- if req, err = http.NewRequest(method, u.String(), nil); err != nil {
- return nil, err
- }
- } else {
- if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil {
- return nil, err
- }
- req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
- }
-
- // Add api token header
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken))
- if c.External {
- req.Header.Add("X-External-Client", "1")
- }
-
- resp, err = c.Client.Do(req)
- if err != nil {
- if retryable {
- time.Sleep(RetryDelay)
- continue
- } else {
- return nil, err
- }
- }
-
- if resp.StatusCode == http.StatusOK {
- return resp.Body, nil
+ if method == "GET" || method == "HEAD" {
+ u.RawQuery = vals.Encode()
+ if req, err = http.NewRequest(method, u.String(), nil); err != nil {
+ return nil, err
}
-
- defer resp.Body.Close()
-
- switch resp.StatusCode {
- case 408, 409, 422, 423, 500, 502, 503, 504:
- time.Sleep(RetryDelay)
- continue
- default:
- return nil, newAPIServerError(c.ApiServer, resp)
+ } else {
+ if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil {
+ return nil, err
}
+ req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
}
-
- if resp != nil {
+ if c.RequestID != "" {
+ req.Header.Add("X-Request-Id", c.RequestID)
+ }
+ client := arvados.Client{
+ Client: c.Client,
+ APIHost: c.ApiServer,
+ AuthToken: c.ApiToken,
+ Insecure: c.ApiInsecure,
+ Timeout: 30 * RetryDelay * time.Duration(c.Retries),
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
+ defer resp.Body.Close()
return nil, newAPIServerError(c.ApiServer, resp)
}
- return nil, err
+ return resp.Body, nil
}
func newAPIServerError(ServerAddress string, resp *http.Response) APIServerError {
// Call an API endpoint and parse the JSON response into an object.
//
-// method - HTTP method: GET, HEAD, PUT, POST, PATCH or DELETE.
-// resourceType - the type of arvados resource to act on (e.g., "collections", "pipeline_instances").
-// uuid - the uuid of the specific item to access. May be empty.
-// action - API method name (e.g., "lock"). This is often empty if implied by method and uuid.
-// parameters - method parameters.
-// output - a map or annotated struct which is a legal target for encoding/json/Decoder.
+// method - HTTP method: GET, HEAD, PUT, POST, PATCH or DELETE.
+// resourceType - the type of arvados resource to act on (e.g., "collections", "pipeline_instances").
+// uuid - the uuid of the specific item to access. May be empty.
+// action - API method name (e.g., "lock"). This is often empty if implied by method and uuid.
+// parameters - method parameters.
+// output - a map or annotated struct which is a legal target for encoding/json/Decoder.
//
// Returns a non-nil error if an error occurs making the API call, the
// API responds with a non-successful HTTP status, or an error occurs
return c.Call("DELETE", resource, uuid, "", parameters, output)
}
-// Modify attributes of a resource. See Call for argument descriptions.
+// Update attributes of a resource. See Call for argument descriptions.
func (c *ArvadosClient) Update(resourceType string, uuid string, parameters Dict, output interface{}) (err error) {
return c.Call("PUT", resourceType, uuid, "", parameters, output)
}
return c.Call("GET", resource, "", "", parameters, output)
}
-const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest"
+const ApiDiscoveryResource = "discovery/v1/apis/arvados/v1/rest"
// Discovery returns the value of the given parameter in the discovery
// document. Returns a non-nil error if the discovery document cannot
func (c *ArvadosClient) Discovery(parameter string) (value interface{}, err error) {
if len(c.DiscoveryDoc) == 0 {
c.DiscoveryDoc = make(Dict)
- err = c.Call("GET", API_DISCOVERY_RESOURCE, "", "", nil, &c.DiscoveryDoc)
+ err = c.Call("GET", ApiDiscoveryResource, "", "", nil, &c.DiscoveryDoc)
if err != nil {
return nil, err
}
value, found = c.DiscoveryDoc[parameter]
if found {
return value, nil
- } else {
- return value, ErrInvalidArgument
}
+ return value, ErrInvalidArgument
+}
+
+// ClusterConfig returns the value of the given key in the current cluster's
+// exported config. If key is an empty string, it'll return the entire config.
+func (c *ArvadosClient) ClusterConfig(key string) (config interface{}, err error) {
+ var clusterConfig interface{}
+ err = c.Call("GET", "config", "", "", nil, &clusterConfig)
+ if err != nil {
+ return nil, err
+ }
+ if key == "" {
+ return clusterConfig, nil
+ }
+ configData, ok := clusterConfig.(map[string]interface{})[key]
+ if !ok {
+ return nil, ErrInvalidArgument
+ }
+ return configData, nil
+}
+
+func (c *ArvadosClient) httpClient() *http.Client {
+ if c.Client != nil {
+ return c.Client
+ }
+ cl := &defaultSecureHTTPClient
+ if c.ApiInsecure {
+ cl = &defaultInsecureHTTPClient
+ }
+ if *cl == nil {
+ defaultHTTPClientMtx.Lock()
+ defer defaultHTTPClientMtx.Unlock()
+ *cl = &http.Client{Transport: &http.Transport{
+ TLSClientConfig: MakeTLSConfig(c.ApiInsecure)}}
+ }
+ return *cl
}