import (
"context"
+ "encoding/json"
+ "errors"
"fmt"
+ "io/ioutil"
+ "mime"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync"
+ "time"
"git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/lib/controller/federation"
"git.arvados.org/arvados.git/lib/controller/router"
"git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
insecureClient *http.Client
dbConnector ctrlctx.DBConnector
limitLogCreate chan struct{}
+
+ cache map[string]*cacheEnt
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
h.proxy = &proxy{
Name: "arvados-controller",
}
+ h.cache = map[string]*cacheEnt{
+ "/discovery/v1/apis/arvados/v1/rest": &cacheEnt{validate: validateDiscoveryDoc},
+ }
go h.trashSweepWorker()
go h.containerLogSweepWorker()
next.ServeHTTP(w, req)
}
+// cacheEnt implements a basic stale-while-revalidate cache, suitable
+// for the Arvados discovery document.
+type cacheEnt struct {
+ validate func(body []byte) error
+ mtx sync.Mutex
+ header http.Header
+ body []byte
+ expireAfter time.Time
+ refreshAfter time.Time
+ refreshLock sync.Mutex
+}
+
+const (
+ cacheTTL = 5 * time.Minute
+ cacheExpire = 24 * time.Hour
+)
+
+func (ent *cacheEnt) refresh(path string, do func(*http.Request) (*http.Response, error)) (http.Header, []byte, error) {
+ ent.refreshLock.Lock()
+ defer ent.refreshLock.Unlock()
+ if header, body, needRefresh := ent.response(); !needRefresh {
+ // another goroutine refreshed successfully while we
+ // were waiting for refreshLock
+ return header, body, nil
+ } else if body != nil {
+ // Cache is present, but expired. We'll try to refresh
+ // below. Meanwhile, other refresh() calls will queue
+ // up for refreshLock -- and we don't want them to
+ // turn into N upstream requests, even if upstream is
+ // failing. (If we succeed we'll update the expiry
+ // time again below with the real cacheTTL -- this
+ // just takes care of the error case.)
+ ent.mtx.Lock()
+ ent.refreshAfter = time.Now().Add(time.Second)
+ ent.mtx.Unlock()
+ }
+
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
+ defer cancel()
+ // 0.0.0.0:0 is just a placeholder here -- do(), which is
+ // localClusterRequest(), will replace the scheme and host
+ // parts with the real proxy destination.
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://0.0.0.0:0/"+path, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+ resp, err := do(req)
+ if err != nil {
+ return nil, nil, err
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, nil, fmt.Errorf("HTTP status %d", resp.StatusCode)
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, nil, fmt.Errorf("Read error: %w", err)
+ }
+ header := http.Header{}
+ for k, v := range resp.Header {
+ if !dropHeaders[k] && k != "X-Request-Id" {
+ header[k] = v
+ }
+ }
+ if ent.validate != nil {
+ if err := ent.validate(body); err != nil {
+ return nil, nil, err
+ }
+ } else if mediatype, _, err := mime.ParseMediaType(header.Get("Content-Type")); err == nil && mediatype == "application/json" {
+ if !json.Valid(body) {
+ return nil, nil, errors.New("invalid JSON encoding in response")
+ }
+ }
+ ent.mtx.Lock()
+ defer ent.mtx.Unlock()
+ ent.header = header
+ ent.body = body
+ ent.refreshAfter = time.Now().Add(cacheTTL)
+ ent.expireAfter = time.Now().Add(cacheExpire)
+ return ent.header, ent.body, nil
+}
+
+func (ent *cacheEnt) response() (http.Header, []byte, bool) {
+ ent.mtx.Lock()
+ defer ent.mtx.Unlock()
+ if ent.expireAfter.Before(time.Now()) {
+ ent.header, ent.body, ent.refreshAfter = nil, nil, time.Time{}
+ }
+ return ent.header, ent.body, ent.refreshAfter.Before(time.Now())
+}
+
+func (ent *cacheEnt) ServeHTTP(ctx context.Context, w http.ResponseWriter, path string, do func(*http.Request) (*http.Response, error)) {
+ header, body, needRefresh := ent.response()
+ if body == nil {
+ // need to fetch before we can return anything
+ var err error
+ header, body, err = ent.refresh(path, do)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ } else if needRefresh {
+ // re-fetch in background
+ go func() {
+ _, _, err := ent.refresh(path, do)
+ if err != nil {
+ ctxlog.FromContext(ctx).WithError(err).WithField("path", path).Warn("error refreshing cache")
+ }
+ }()
+ }
+ for k, v := range header {
+ w.Header()[k] = v
+ }
+ w.WriteHeader(http.StatusOK)
+ w.Write(body)
+}
+
func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) {
+ if ent, ok := h.cache[req.URL.Path]; ok && req.Method == http.MethodGet {
+ ent.ServeHTTP(req.Context(), w, req.URL.Path, h.localClusterRequest)
+ return
+ }
resp, err := h.localClusterRequest(req)
n, err := h.proxy.ForwardResponse(w, resp, err)
if err != nil {
}
return best, cluster.TLS.Insecure, nil
}
+
+func validateDiscoveryDoc(body []byte) error {
+ var dd arvados.DiscoveryDocument
+ err := json.Unmarshal(body, &dd)
+ if err != nil {
+ return fmt.Errorf("error decoding JSON response: %w", err)
+ }
+ if dd.BasePath == "" {
+ return errors.New("error in discovery document: no value for basePath")
+ }
+ return nil
+}
"net/url"
"os"
"strings"
+ "sync"
"testing"
"time"
var _ = check.Suite(&HandlerSuite{})
type HandlerSuite struct {
- cluster *arvados.Cluster
- handler *Handler
- logbuf *bytes.Buffer
- ctx context.Context
- cancel context.CancelFunc
+ cluster *arvados.Cluster
+ handler *Handler
+ railsSpy *arvadostest.Proxy
+ logbuf *bytes.Buffer
+ ctx context.Context
+ cancel context.CancelFunc
}
func (s *HandlerSuite) SetUpTest(c *check.C) {
s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
s.cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, s.railsSpy.URL.String())
arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry()).(*Handler)
}
}
}
+func (s *HandlerSuite) TestDiscoveryDocCache(c *check.C) {
+ countRailsReqs := func() int {
+ n := 0
+ for _, req := range s.railsSpy.RequestDumps {
+ if bytes.Contains(req, []byte("/discovery/v1/apis/arvados/v1/rest")) {
+ n++
+ }
+ }
+ return n
+ }
+ getDD := func() int {
+ req := httptest.NewRequest(http.MethodGet, "/discovery/v1/apis/arvados/v1/rest", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ if resp.Code == http.StatusOK {
+ var dd arvados.DiscoveryDocument
+ err := json.Unmarshal(resp.Body.Bytes(), &dd)
+ c.Check(err, check.IsNil)
+ c.Check(dd.Schemas["Collection"].UUIDPrefix, check.Equals, "4zz18")
+ }
+ return resp.Code
+ }
+ getDDConcurrently := func(n int, expectCode int, checkArgs ...interface{}) *sync.WaitGroup {
+ var wg sync.WaitGroup
+ for i := 0; i < n; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ c.Check(getDD(), check.Equals, append([]interface{}{expectCode}, checkArgs...)...)
+ }()
+ }
+ return &wg
+ }
+ clearCache := func() {
+ for _, ent := range s.handler.cache {
+ ent.refreshLock.Lock()
+ ent.mtx.Lock()
+ ent.body, ent.header, ent.refreshAfter = nil, nil, time.Time{}
+ ent.mtx.Unlock()
+ ent.refreshLock.Unlock()
+ }
+ }
+ waitPendingUpdates := func() {
+ for _, ent := range s.handler.cache {
+ ent.refreshLock.Lock()
+ defer ent.refreshLock.Unlock()
+ ent.mtx.Lock()
+ defer ent.mtx.Unlock()
+ }
+ }
+ refreshNow := func() {
+ waitPendingUpdates()
+ for _, ent := range s.handler.cache {
+ ent.refreshAfter = time.Now()
+ }
+ }
+ expireNow := func() {
+ waitPendingUpdates()
+ for _, ent := range s.handler.cache {
+ ent.expireAfter = time.Now()
+ }
+ }
+
+ // Easy path: first req fetches, subsequent reqs use cache.
+ c.Check(countRailsReqs(), check.Equals, 0)
+ c.Check(getDD(), check.Equals, http.StatusOK)
+ c.Check(countRailsReqs(), check.Equals, 1)
+ c.Check(getDD(), check.Equals, http.StatusOK)
+ c.Check(countRailsReqs(), check.Equals, 1)
+ c.Check(getDD(), check.Equals, http.StatusOK)
+ c.Check(countRailsReqs(), check.Equals, 1)
+
+ // To guarantee we have concurrent requests, we set up
+ // railsSpy to hold up the Handler's outgoing requests until
+ // we send to (or close) holdReqs.
+ holdReqs := make(chan struct{})
+ s.railsSpy.Director = func(*http.Request) {
+ <-holdReqs
+ }
+
+ // Race at startup: first req fetches, other concurrent reqs
+ // wait for the initial fetch to complete, then all return.
+ clearCache()
+ reqsBefore := countRailsReqs()
+ wg := getDDConcurrently(5, http.StatusOK, check.Commentf("race at startup"))
+ close(holdReqs)
+ wg.Wait()
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
+
+ // Race after expiry: concurrent reqs return the cached data
+ // but initiate a new fetch in the background.
+ refreshNow()
+ holdReqs = make(chan struct{})
+ wg = getDDConcurrently(5, http.StatusOK, check.Commentf("race after expiry"))
+ reqsBefore = countRailsReqs()
+ close(holdReqs)
+ wg.Wait()
+ for deadline := time.Now().Add(time.Second); time.Now().Before(deadline) && countRailsReqs() < reqsBefore+1; {
+ time.Sleep(time.Second / 100)
+ }
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
+
+ // Configure railsSpy to return an error or bad content
+ // depending on flags.
+ var wantError, wantBadContent bool
+ s.railsSpy.Director = func(req *http.Request) {
+ if wantError {
+ req.Method = "MAKE-COFFEE"
+ } else if wantBadContent {
+ req.URL.Path = "/_health/ping"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
+ }
+ }
+
+ // Error at startup (empty cache) => caller gets error, and we
+ // make an upstream attempt for each incoming request because
+ // we have nothing better to return
+ clearCache()
+ wantError, wantBadContent = true, false
+ reqsBefore = countRailsReqs()
+ holdReqs = make(chan struct{})
+ wg = getDDConcurrently(5, http.StatusBadGateway, check.Commentf("error at startup"))
+ close(holdReqs)
+ wg.Wait()
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+5)
+
+ // Response status is OK but body is not a discovery document
+ wantError, wantBadContent = false, true
+ reqsBefore = countRailsReqs()
+ c.Check(getDD(), check.Equals, http.StatusBadGateway)
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
+
+ // Error condition clears => caller gets OK, cache is warmed
+ // up
+ wantError, wantBadContent = false, false
+ reqsBefore = countRailsReqs()
+ getDDConcurrently(5, http.StatusOK, check.Commentf("success after errors at startup")).Wait()
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
+
+ // Error with warm cache => caller gets OK (with no attempt to
+ // re-fetch)
+ wantError, wantBadContent = true, false
+ reqsBefore = countRailsReqs()
+ getDDConcurrently(5, http.StatusOK, check.Commentf("error with warm cache")).Wait()
+ c.Check(countRailsReqs(), check.Equals, reqsBefore)
+
+ // Error with stale cache => caller gets OK with stale data
+ // while the re-fetch is attempted in the background
+ refreshNow()
+ wantError, wantBadContent = true, false
+ reqsBefore = countRailsReqs()
+ holdReqs = make(chan struct{})
+ getDDConcurrently(5, http.StatusOK, check.Commentf("error with stale cache")).Wait()
+ close(holdReqs)
+ // Only one attempt to re-fetch (holdReqs ensured the first
+ // update took long enough for the last incoming request to
+ // arrive)
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
+
+ refreshNow()
+ wantError, wantBadContent = false, false
+ reqsBefore = countRailsReqs()
+ holdReqs = make(chan struct{})
+ getDDConcurrently(5, http.StatusOK, check.Commentf("refresh cache after error condition clears")).Wait()
+ close(holdReqs)
+ waitPendingUpdates()
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
+
+ // Make sure expireAfter is getting set
+ waitPendingUpdates()
+ exp := s.handler.cache["/discovery/v1/apis/arvados/v1/rest"].expireAfter.Sub(time.Now())
+ c.Check(exp > cacheTTL, check.Equals, true)
+ c.Check(exp < cacheExpire, check.Equals, true)
+
+ // After the cache *expires* it behaves as if uninitialized:
+ // each incoming request does a new upstream request until one
+ // succeeds.
+ //
+ // First check failure after expiry:
+ expireNow()
+ wantError, wantBadContent = true, false
+ reqsBefore = countRailsReqs()
+ holdReqs = make(chan struct{})
+ wg = getDDConcurrently(5, http.StatusBadGateway, check.Commentf("error after expiry"))
+ close(holdReqs)
+ wg.Wait()
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+5)
+
+ // Success after expiry:
+ wantError, wantBadContent = false, false
+ reqsBefore = countRailsReqs()
+ holdReqs = make(chan struct{})
+ wg = getDDConcurrently(5, http.StatusOK, check.Commentf("success after expiry"))
+ close(holdReqs)
+ wg.Wait()
+ c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
+}
+
func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
voc := `{
"strict_tags": false,
// etc.
func (s *HandlerSuite) TestRequestCancel(c *check.C) {
ctx, cancel := context.WithCancel(context.Background())
- req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil).WithContext(ctx)
+ req := httptest.NewRequest("GET", "/static/login_failure", nil).WithContext(ctx)
resp := httptest.NewRecorder()
cancel()
s.handler.ServeHTTP(resp, req)
protected
def discovery_doc
- Rails.cache.fetch 'arvados_v1_rest_discovery' do
- Rails.application.eager_load!
- remoteHosts = {}
- Rails.configuration.RemoteClusters.each {|k,v| if k != :"*" then remoteHosts[k] = v["Host"] end }
- discovery = {
- kind: "discovery#restDescription",
- discoveryVersion: "v1",
- id: "arvados:v1",
- name: "arvados",
- version: "v1",
- # format is YYYYMMDD, must be fixed width (needs to be lexically
- # sortable), updated manually, may be used by clients to
- # determine availability of API server features.
- revision: "20220510",
- source_version: AppVersion.hash,
- sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
- packageVersion: AppVersion.package_version,
- generatedAt: db_current_time.iso8601,
- title: "Arvados API",
- description: "The API to interact with Arvados.",
- documentationLink: "http://doc.arvados.org/api/index.html",
- defaultCollectionReplication: Rails.configuration.Collections.DefaultReplication,
- protocol: "rest",
- baseUrl: root_url + "arvados/v1/",
- basePath: "/arvados/v1/",
- rootUrl: root_url,
- servicePath: "arvados/v1/",
- batchPath: "batch",
- uuidPrefix: Rails.configuration.ClusterID,
- defaultTrashLifetime: Rails.configuration.Collections.DefaultTrashLifetime,
- blobSignatureTtl: Rails.configuration.Collections.BlobSigningTTL,
- maxRequestSize: Rails.configuration.API.MaxRequestSize,
- maxItemsPerResponse: Rails.configuration.API.MaxItemsPerResponse,
- dockerImageFormats: Rails.configuration.Containers.SupportedDockerImageFormats.keys,
- crunchLogBytesPerEvent: Rails.configuration.Containers.Logging.LogBytesPerEvent,
- crunchLogSecondsBetweenEvents: Rails.configuration.Containers.Logging.LogSecondsBetweenEvents,
- crunchLogThrottlePeriod: Rails.configuration.Containers.Logging.LogThrottlePeriod,
- crunchLogThrottleBytes: Rails.configuration.Containers.Logging.LogThrottleBytes,
- crunchLogThrottleLines: Rails.configuration.Containers.Logging.LogThrottleLines,
- crunchLimitLogBytesPerJob: Rails.configuration.Containers.Logging.LimitLogBytesPerJob,
- crunchLogPartialLineThrottlePeriod: Rails.configuration.Containers.Logging.LogPartialLineThrottlePeriod,
- crunchLogUpdatePeriod: Rails.configuration.Containers.Logging.LogUpdatePeriod,
- crunchLogUpdateSize: Rails.configuration.Containers.Logging.LogUpdateSize,
- remoteHosts: remoteHosts,
- remoteHostsViaDNS: Rails.configuration.RemoteClusters["*"].Proxy,
- websocketUrl: Rails.configuration.Services.Websocket.ExternalURL.to_s,
- workbenchUrl: Rails.configuration.Services.Workbench1.ExternalURL.to_s,
- workbench2Url: Rails.configuration.Services.Workbench2.ExternalURL.to_s,
- keepWebServiceUrl: Rails.configuration.Services.WebDAV.ExternalURL.to_s,
- gitUrl: Rails.configuration.Services.GitHTTP.ExternalURL.to_s,
- parameters: {
- alt: {
+ Rails.application.eager_load!
+ remoteHosts = {}
+ Rails.configuration.RemoteClusters.each {|k,v| if k != :"*" then remoteHosts[k] = v["Host"] end }
+ discovery = {
+ kind: "discovery#restDescription",
+ discoveryVersion: "v1",
+ id: "arvados:v1",
+ name: "arvados",
+ version: "v1",
+ # format is YYYYMMDD, must be fixed width (needs to be lexically
+ # sortable), updated manually, may be used by clients to
+ # determine availability of API server features.
+ revision: "20220510",
+ source_version: AppVersion.hash,
+ sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
+ packageVersion: AppVersion.package_version,
+ generatedAt: db_current_time.iso8601,
+ title: "Arvados API",
+ description: "The API to interact with Arvados.",
+ documentationLink: "http://doc.arvados.org/api/index.html",
+ defaultCollectionReplication: Rails.configuration.Collections.DefaultReplication,
+ protocol: "rest",
+ baseUrl: root_url + "arvados/v1/",
+ basePath: "/arvados/v1/",
+ rootUrl: root_url,
+ servicePath: "arvados/v1/",
+ batchPath: "batch",
+ uuidPrefix: Rails.configuration.ClusterID,
+ defaultTrashLifetime: Rails.configuration.Collections.DefaultTrashLifetime,
+ blobSignatureTtl: Rails.configuration.Collections.BlobSigningTTL,
+ maxRequestSize: Rails.configuration.API.MaxRequestSize,
+ maxItemsPerResponse: Rails.configuration.API.MaxItemsPerResponse,
+ dockerImageFormats: Rails.configuration.Containers.SupportedDockerImageFormats.keys,
+ crunchLogBytesPerEvent: Rails.configuration.Containers.Logging.LogBytesPerEvent,
+ crunchLogSecondsBetweenEvents: Rails.configuration.Containers.Logging.LogSecondsBetweenEvents,
+ crunchLogThrottlePeriod: Rails.configuration.Containers.Logging.LogThrottlePeriod,
+ crunchLogThrottleBytes: Rails.configuration.Containers.Logging.LogThrottleBytes,
+ crunchLogThrottleLines: Rails.configuration.Containers.Logging.LogThrottleLines,
+ crunchLimitLogBytesPerJob: Rails.configuration.Containers.Logging.LimitLogBytesPerJob,
+ crunchLogPartialLineThrottlePeriod: Rails.configuration.Containers.Logging.LogPartialLineThrottlePeriod,
+ crunchLogUpdatePeriod: Rails.configuration.Containers.Logging.LogUpdatePeriod,
+ crunchLogUpdateSize: Rails.configuration.Containers.Logging.LogUpdateSize,
+ remoteHosts: remoteHosts,
+ remoteHostsViaDNS: Rails.configuration.RemoteClusters["*"].Proxy,
+ websocketUrl: Rails.configuration.Services.Websocket.ExternalURL.to_s,
+ workbenchUrl: Rails.configuration.Services.Workbench1.ExternalURL.to_s,
+ workbench2Url: Rails.configuration.Services.Workbench2.ExternalURL.to_s,
+ keepWebServiceUrl: Rails.configuration.Services.WebDAV.ExternalURL.to_s,
+ gitUrl: Rails.configuration.Services.GitHTTP.ExternalURL.to_s,
+ parameters: {
+ alt: {
+ type: "string",
+ description: "Data format for the response.",
+ default: "json",
+ enum: [
+ "json"
+ ],
+ enumDescriptions: [
+ "Responses with Content-Type of application/json"
+ ],
+ location: "query"
+ },
+ fields: {
+ type: "string",
+ description: "Selector specifying which fields to include in a partial response.",
+ location: "query"
+ },
+ key: {
+ type: "string",
+ description: "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
+ location: "query"
+ },
+ oauth_token: {
+ type: "string",
+ description: "OAuth 2.0 token for the current user.",
+ location: "query"
+ }
+ },
+ auth: {
+ oauth2: {
+ scopes: {
+ "https://api.arvados.org/auth/arvados" => {
+ description: "View and manage objects"
+ },
+ "https://api.arvados.org/auth/arvados.readonly" => {
+ description: "View objects"
+ }
+ }
+ }
+ },
+ schemas: {},
+ resources: {}
+ }
+
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
+ begin
+ ctl_class = "Arvados::V1::#{k.to_s.pluralize}Controller".constantize
+ rescue
+ # No controller -> no discovery.
+ next
+ end
+ object_properties = {}
+ k.columns.
+ select { |col| col.name != 'id' && !col.name.start_with?('secret_') }.
+ collect do |col|
+ if k.serialized_attributes.has_key? col.name
+ object_properties[col.name] = {
+ type: k.serialized_attributes[col.name].object_class.to_s
+ }
+ elsif k.attribute_types[col.name].is_a? JsonbType::Hash
+ object_properties[col.name] = {
+ type: Hash.to_s
+ }
+ elsif k.attribute_types[col.name].is_a? JsonbType::Array
+ object_properties[col.name] = {
+ type: Array.to_s
+ }
+ else
+ object_properties[col.name] = {
+ type: col.type
+ }
+ end
+ end
+ discovery[:schemas][k.to_s + 'List'] = {
+ id: k.to_s + 'List',
+ description: k.to_s + ' list',
+ type: "object",
+ properties: {
+ kind: {
type: "string",
- description: "Data format for the response.",
- default: "json",
- enum: [
- "json"
- ],
- enumDescriptions: [
- "Responses with Content-Type of application/json"
- ],
- location: "query"
+ description: "Object type. Always arvados##{k.to_s.camelcase(:lower)}List.",
+ default: "arvados##{k.to_s.camelcase(:lower)}List"
+ },
+ etag: {
+ type: "string",
+ description: "List version."
+ },
+ items: {
+ type: "array",
+ description: "The list of #{k.to_s.pluralize}.",
+ items: {
+ "$ref" => k.to_s
+ }
},
- fields: {
+ next_link: {
type: "string",
- description: "Selector specifying which fields to include in a partial response.",
- location: "query"
+ description: "A link to the next page of #{k.to_s.pluralize}."
},
- key: {
+ next_page_token: {
type: "string",
- description: "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
- location: "query"
+ description: "The page token for the next page of #{k.to_s.pluralize}."
},
- oauth_token: {
+ selfLink: {
type: "string",
- description: "OAuth 2.0 token for the current user.",
- location: "query"
+ description: "A link back to this list."
}
- },
- auth: {
- oauth2: {
- scopes: {
- "https://api.arvados.org/auth/arvados" => {
- description: "View and manage objects"
- },
- "https://api.arvados.org/auth/arvados.readonly" => {
- description: "View objects"
- }
- }
+ }
+ }
+ discovery[:schemas][k.to_s] = {
+ id: k.to_s,
+ description: k.to_s,
+ type: "object",
+ uuidPrefix: (k.respond_to?(:uuid_prefix) ? k.uuid_prefix : nil),
+ properties: {
+ uuid: {
+ type: "string",
+ description: "Object ID."
+ },
+ etag: {
+ type: "string",
+ description: "Object version."
}
- },
- schemas: {},
- resources: {}
+ }.merge(object_properties)
}
-
- ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
- begin
- ctl_class = "Arvados::V1::#{k.to_s.pluralize}Controller".constantize
- rescue
- # No controller -> no discovery.
- next
- end
- object_properties = {}
- k.columns.
- select { |col| col.name != 'id' && !col.name.start_with?('secret_') }.
- collect do |col|
- if k.serialized_attributes.has_key? col.name
- object_properties[col.name] = {
- type: k.serialized_attributes[col.name].object_class.to_s
- }
- elsif k.attribute_types[col.name].is_a? JsonbType::Hash
- object_properties[col.name] = {
- type: Hash.to_s
- }
- elsif k.attribute_types[col.name].is_a? JsonbType::Array
- object_properties[col.name] = {
- type: Array.to_s
- }
- else
- object_properties[col.name] = {
- type: col.type
- }
- end
- end
- discovery[:schemas][k.to_s + 'List'] = {
- id: k.to_s + 'List',
- description: k.to_s + ' list',
- type: "object",
- properties: {
- kind: {
- type: "string",
- description: "Object type. Always arvados##{k.to_s.camelcase(:lower)}List.",
- default: "arvados##{k.to_s.camelcase(:lower)}List"
- },
- etag: {
- type: "string",
- description: "List version."
- },
- items: {
- type: "array",
- description: "The list of #{k.to_s.pluralize}.",
- items: {
- "$ref" => k.to_s
+ discovery[:resources][k.to_s.underscore.pluralize] = {
+ methods: {
+ get: {
+ id: "arvados.#{k.to_s.underscore.pluralize}.get",
+ path: "#{k.to_s.underscore.pluralize}/{uuid}",
+ httpMethod: "GET",
+ description: "Gets a #{k.to_s}'s metadata by UUID.",
+ parameters: {
+ uuid: {
+ type: "string",
+ description: "The UUID of the #{k.to_s} in question.",
+ required: true,
+ location: "path"
}
},
- next_link: {
- type: "string",
- description: "A link to the next page of #{k.to_s.pluralize}."
- },
- next_page_token: {
- type: "string",
- description: "The page token for the next page of #{k.to_s.pluralize}."
- },
- selfLink: {
- type: "string",
- description: "A link back to this list."
- }
- }
- }
- discovery[:schemas][k.to_s] = {
- id: k.to_s,
- description: k.to_s,
- type: "object",
- uuidPrefix: (k.respond_to?(:uuid_prefix) ? k.uuid_prefix : nil),
- properties: {
- uuid: {
- type: "string",
- description: "Object ID."
- },
- etag: {
- type: "string",
- description: "Object version."
- }
- }.merge(object_properties)
- }
- discovery[:resources][k.to_s.underscore.pluralize] = {
- methods: {
- get: {
- id: "arvados.#{k.to_s.underscore.pluralize}.get",
- path: "#{k.to_s.underscore.pluralize}/{uuid}",
- httpMethod: "GET",
- description: "Gets a #{k.to_s}'s metadata by UUID.",
- parameters: {
- uuid: {
- type: "string",
- description: "The UUID of the #{k.to_s} in question.",
- required: true,
- location: "path"
- }
- },
- parameterOrder: [
- "uuid"
- ],
- response: {
- "$ref" => k.to_s
- },
- scopes: [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
+ parameterOrder: [
+ "uuid"
+ ],
+ response: {
+ "$ref" => k.to_s
},
- index: {
- id: "arvados.#{k.to_s.underscore.pluralize}.index",
- path: k.to_s.underscore.pluralize,
- httpMethod: "GET",
- description:
- %|Index #{k.to_s.pluralize}.
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
+ index: {
+ id: "arvados.#{k.to_s.underscore.pluralize}.index",
+ path: k.to_s.underscore.pluralize,
+ httpMethod: "GET",
+ description:
+ %|Index #{k.to_s.pluralize}.
The <code>index</code> method returns a
<a href="/api/resources.html">resource list</a> of
"request_time":0.157236317
}
</pre>|,
- parameters: {
- },
- response: {
- "$ref" => "#{k.to_s}List"
- },
- scopes: [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
+ parameters: {
},
- create: {
- id: "arvados.#{k.to_s.underscore.pluralize}.create",
- path: "#{k.to_s.underscore.pluralize}",
- httpMethod: "POST",
- description: "Create a new #{k.to_s}.",
- parameters: {},
- request: {
- required: true,
- properties: {
- k.to_s.underscore => {
- "$ref" => k.to_s
- }
- }
- },
- response: {
- "$ref" => k.to_s
- },
- scopes: [
- "https://api.arvados.org/auth/arvados"
- ]
+ response: {
+ "$ref" => "#{k.to_s}List"
},
- update: {
- id: "arvados.#{k.to_s.underscore.pluralize}.update",
- path: "#{k.to_s.underscore.pluralize}/{uuid}",
- httpMethod: "PUT",
- description: "Update attributes of an existing #{k.to_s}.",
- parameters: {
- uuid: {
- type: "string",
- description: "The UUID of the #{k.to_s} in question.",
- required: true,
- location: "path"
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
+ create: {
+ id: "arvados.#{k.to_s.underscore.pluralize}.create",
+ path: "#{k.to_s.underscore.pluralize}",
+ httpMethod: "POST",
+ description: "Create a new #{k.to_s}.",
+ parameters: {},
+ request: {
+ required: true,
+ properties: {
+ k.to_s.underscore => {
+ "$ref" => k.to_s
}
- },
- request: {
+ }
+ },
+ response: {
+ "$ref" => k.to_s
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados"
+ ]
+ },
+ update: {
+ id: "arvados.#{k.to_s.underscore.pluralize}.update",
+ path: "#{k.to_s.underscore.pluralize}/{uuid}",
+ httpMethod: "PUT",
+ description: "Update attributes of an existing #{k.to_s}.",
+ parameters: {
+ uuid: {
+ type: "string",
+ description: "The UUID of the #{k.to_s} in question.",
required: true,
- properties: {
- k.to_s.underscore => {
- "$ref" => k.to_s
- }
+ location: "path"
+ }
+ },
+ request: {
+ required: true,
+ properties: {
+ k.to_s.underscore => {
+ "$ref" => k.to_s
}
- },
+ }
+ },
+ response: {
+ "$ref" => k.to_s
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados"
+ ]
+ },
+ delete: {
+ id: "arvados.#{k.to_s.underscore.pluralize}.delete",
+ path: "#{k.to_s.underscore.pluralize}/{uuid}",
+ httpMethod: "DELETE",
+ description: "Delete an existing #{k.to_s}.",
+ parameters: {
+ uuid: {
+ type: "string",
+ description: "The UUID of the #{k.to_s} in question.",
+ required: true,
+ location: "path"
+ }
+ },
+ response: {
+ "$ref" => k.to_s
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados"
+ ]
+ }
+ }
+ }
+ # Check for Rails routes that don't match the usual actions
+ # listed above
+ d_methods = discovery[:resources][k.to_s.underscore.pluralize][:methods]
+ Rails.application.routes.routes.each do |route|
+ action = route.defaults[:action]
+ httpMethod = ['GET', 'POST', 'PUT', 'DELETE'].map { |method|
+ method if route.verb.match(method)
+ }.compact.first
+ if httpMethod and
+ route.defaults[:controller] == 'arvados/v1/' + k.to_s.underscore.pluralize and
+ ctl_class.action_methods.include? action
+ if !d_methods[action.to_sym]
+ method = {
+ id: "arvados.#{k.to_s.underscore.pluralize}.#{action}",
+ path: route.path.spec.to_s.sub('/arvados/v1/','').sub('(.:format)','').sub(/:(uu)?id/,'{uuid}'),
+ httpMethod: httpMethod,
+ description: "#{action} #{k.to_s.underscore.pluralize}",
+ parameters: {},
response: {
- "$ref" => k.to_s
+ "$ref" => (action == 'index' ? "#{k.to_s}List" : k.to_s)
},
scopes: [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- delete: {
- id: "arvados.#{k.to_s.underscore.pluralize}.delete",
- path: "#{k.to_s.underscore.pluralize}/{uuid}",
- httpMethod: "DELETE",
- description: "Delete an existing #{k.to_s}.",
- parameters: {
- uuid: {
+ "https://api.arvados.org/auth/arvados"
+ ]
+ }
+ route.segment_keys.each do |key|
+ if key != :format
+ key = :uuid if key == :id
+ method[:parameters][key] = {
type: "string",
- description: "The UUID of the #{k.to_s} in question.",
+ description: "",
required: true,
location: "path"
}
- },
- response: {
- "$ref" => k.to_s
- },
- scopes: [
- "https://api.arvados.org/auth/arvados"
- ]
- }
- }
- }
- # Check for Rails routes that don't match the usual actions
- # listed above
- d_methods = discovery[:resources][k.to_s.underscore.pluralize][:methods]
- Rails.application.routes.routes.each do |route|
- action = route.defaults[:action]
- httpMethod = ['GET', 'POST', 'PUT', 'DELETE'].map { |method|
- method if route.verb.match(method)
- }.compact.first
- if httpMethod and
- route.defaults[:controller] == 'arvados/v1/' + k.to_s.underscore.pluralize and
- ctl_class.action_methods.include? action
- if !d_methods[action.to_sym]
- method = {
- id: "arvados.#{k.to_s.underscore.pluralize}.#{action}",
- path: route.path.spec.to_s.sub('/arvados/v1/','').sub('(.:format)','').sub(/:(uu)?id/,'{uuid}'),
- httpMethod: httpMethod,
- description: "#{action} #{k.to_s.underscore.pluralize}",
- parameters: {},
- response: {
- "$ref" => (action == 'index' ? "#{k.to_s}List" : k.to_s)
- },
- scopes: [
- "https://api.arvados.org/auth/arvados"
- ]
- }
- route.segment_keys.each do |key|
- if key != :format
- key = :uuid if key == :id
- method[:parameters][key] = {
- type: "string",
- description: "",
- required: true,
- location: "path"
- }
- end
end
- else
- # We already built a generic method description, but we
- # might find some more required parameters through
- # introspection.
- method = d_methods[action.to_sym]
end
- if ctl_class.respond_to? "_#{action}_requires_parameters".to_sym
- ctl_class.send("_#{action}_requires_parameters".to_sym).each do |l, v|
- if v.is_a? Hash
- method[:parameters][l] = v
- else
- method[:parameters][l] = {}
- end
- if !method[:parameters][l][:default].nil?
- # The JAVA SDK is sensitive to all values being strings
- method[:parameters][l][:default] = method[:parameters][l][:default].to_s
- end
- method[:parameters][l][:type] ||= 'string'
- method[:parameters][l][:description] ||= ''
- method[:parameters][l][:location] = (route.segment_keys.include?(l) ? 'path' : 'query')
- if method[:parameters][l][:required].nil?
- method[:parameters][l][:required] = v != false
- end
+ else
+ # We already built a generic method description, but we
+ # might find some more required parameters through
+ # introspection.
+ method = d_methods[action.to_sym]
+ end
+ if ctl_class.respond_to? "_#{action}_requires_parameters".to_sym
+ ctl_class.send("_#{action}_requires_parameters".to_sym).each do |l, v|
+ if v.is_a? Hash
+ method[:parameters][l] = v
+ else
+ method[:parameters][l] = {}
+ end
+ if !method[:parameters][l][:default].nil?
+ # The JAVA SDK is sensitive to all values being strings
+ method[:parameters][l][:default] = method[:parameters][l][:default].to_s
+ end
+ method[:parameters][l][:type] ||= 'string'
+ method[:parameters][l][:description] ||= ''
+ method[:parameters][l][:location] = (route.segment_keys.include?(l) ? 'path' : 'query')
+ if method[:parameters][l][:required].nil?
+ method[:parameters][l][:required] = v != false
end
end
- d_methods[action.to_sym] = method
+ end
+ d_methods[action.to_sym] = method
- if action == 'index'
- list_method = method.dup
- list_method[:id].sub!('index', 'list')
- list_method[:description].sub!('Index', 'List')
- list_method[:description].sub!('index', 'list')
- d_methods[:list] = list_method
- end
+ if action == 'index'
+ list_method = method.dup
+ list_method[:id].sub!('index', 'list')
+ list_method[:description].sub!('Index', 'List')
+ list_method[:description].sub!('index', 'list')
+ d_methods[:list] = list_method
end
end
end
+ end
- # The 'replace_files' option is implemented in lib/controller,
- # not Rails -- we just need to add it here so discovery-aware
- # clients know how to validate it.
- [:create, :update].each do |action|
- discovery[:resources]['collections'][:methods][action][:parameters]['replace_files'] = {
- type: 'object',
- description: 'Files and directories to initialize/replace with content from other collections.',
- required: false,
- location: 'query',
- properties: {},
- additionalProperties: {type: 'string'},
- }
- end
+ # The 'replace_files' option is implemented in lib/controller,
+ # not Rails -- we just need to add it here so discovery-aware
+ # clients know how to validate it.
+ [:create, :update].each do |action|
+ discovery[:resources]['collections'][:methods][action][:parameters]['replace_files'] = {
+ type: 'object',
+ description: 'Files and directories to initialize/replace with content from other collections.',
+ required: false,
+ location: 'query',
+ properties: {},
+ additionalProperties: {type: 'string'},
+ }
+ end
- discovery[:resources]['configs'] = {
- methods: {
- get: {
- id: "arvados.configs.get",
- path: "config",
- httpMethod: "GET",
- description: "Get public config",
- parameters: {
- },
- parameterOrder: [
- ],
- response: {
- },
- scopes: [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
+ discovery[:resources]['configs'] = {
+ methods: {
+ get: {
+ id: "arvados.configs.get",
+ path: "config",
+ httpMethod: "GET",
+ description: "Get public config",
+ parameters: {
},
- }
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
}
+ }
- discovery[:resources]['vocabularies'] = {
- methods: {
- get: {
- id: "arvados.vocabularies.get",
- path: "vocabulary",
- httpMethod: "GET",
- description: "Get vocabulary definition",
- parameters: {
- },
- parameterOrder: [
- ],
- response: {
- },
- scopes: [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
+ discovery[:resources]['vocabularies'] = {
+ methods: {
+ get: {
+ id: "arvados.vocabularies.get",
+ path: "vocabulary",
+ httpMethod: "GET",
+ description: "Get vocabulary definition",
+ parameters: {
},
- }
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
}
+ }
- discovery[:resources]['sys'] = {
- methods: {
- get: {
- id: "arvados.sys.trash_sweep",
- path: "sys/trash_sweep",
- httpMethod: "POST",
- description: "apply scheduled trash and delete operations",
- parameters: {
- },
- parameterOrder: [
- ],
- response: {
- },
- scopes: [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
+ discovery[:resources]['sys'] = {
+ methods: {
+ get: {
+ id: "arvados.sys.trash_sweep",
+ path: "sys/trash_sweep",
+ httpMethod: "POST",
+ description: "apply scheduled trash and delete operations",
+ parameters: {
},
- }
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
}
+ }
- Rails.configuration.API.DisabledAPIs.each do |method, _|
- ctrl, action = method.to_s.split('.', 2)
- discovery[:resources][ctrl][:methods].delete(action.to_sym)
- end
- discovery
+ Rails.configuration.API.DisabledAPIs.each do |method, _|
+ ctrl, action = method.to_s.split('.', 2)
+ discovery[:resources][ctrl][:methods].delete(action.to_sym)
end
+ discovery
end
end
projects / arvados.git / commitdiff