14087: Federated fetch by PDH WIP

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index da4c00da268bd0b9958fe91137816676aa7c974f..d68f44d95b65ebd7afd2372c55a8b5fbf16d6cda 100644 (file)
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -24,6 +24,7 @@ import (
 
 var wfRe = regexp.MustCompile(`^/arvados/v1/workflows/([0-9a-z]{5})-[^/]+$`)
 var collectionRe = regexp.MustCompile(`^/arvados/v1/collections/([0-9a-z]{5})-[^/]+$`)
+var collectionByPDHRe = regexp.MustCompile(`^/arvados/v1/collections/([0-9a-fA-F]{32}\+[0-9]+)+$`)
 
 type genericFederatedRequestHandler struct {
        next    http.Handler
@@ -132,8 +133,49 @@ func (clusterId rewriteSignaturesClusterId) rewriteSignatures(resp *http.Respons
        return resp, nil
 }
 
+type searchLocalClusterForPDH struct {
+       needSearchFederation bool
+}
+
+func (s *searchLocalClusterForPDH) filterLocalClusterResponse(resp *http.Response) (newResponse *http.Response, err error) {
+       if resp.StatusCode == 404 {
+               // Suppress returning this result, because we want to
+               // search the federation.
+               s.needSearchFederation = true
+               return nil, nil
+       }
+       return resp, nil
+}
+
 func (h *collectionFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-       m := collectionRe.FindStringSubmatch(req.URL.Path)
+       m := collectionByPDHRe.FindStringSubmatch(req.URL.Path)
+       if len(m) == 2 {
+               urlOut, insecure, err := findRailsAPI(h.handler.Cluster, h.handler.NodeProfile)
+               if err != nil {
+                       httpserver.Error(w, err.Error(), http.StatusInternalServerError)
+                       return
+               }
+
+               urlOut = &url.URL{
+                       Scheme:   urlOut.Scheme,
+                       Host:     urlOut.Host,
+                       Path:     req.URL.Path,
+                       RawPath:  req.URL.RawPath,
+                       RawQuery: req.URL.RawQuery,
+               }
+               client := h.handler.secureClient
+               if insecure {
+                       client = h.handler.insecureClient
+               }
+               sf := &searchLocalClusterForPDH{false}
+               h.handler.proxy.Do(w, req, urlOut, client, sf.filterLocalClusterResponse)
+               if !sf.needSearchFederation {
+                       // a response was sent
+                       return
+               }
+       }
+
+       m = collectionRe.FindStringSubmatch(req.URL.Path)
        if len(m) < 2 || m[1] == h.handler.Cluster.ClusterID {
                h.next.ServeHTTP(w, req)
                return

diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 8d32b695da94ec3cdadbd30a827bff24c03234e2..53c6b9518a1f303396448b883299805419ab72e6 100644 (file)
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -335,7 +335,9 @@ func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
        c.Check(col.ManifestText, check.Matches,
                `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 `)
+}
 
+func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
        // Confirm the regular expression identifies other groups of hints correctly
        c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
                check.DeepEquals,
@@ -347,3 +349,56 @@ func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
                        "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
                        "+D3+E4", "+E4"})
 }
+
+func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
+       np := arvados.NodeProfile{
+               Controller: arvados.SystemServiceInstance{Listen: ":"},
+               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
+                       TLS: true, Insecure: true}}
+       s.testHandler.Cluster.NodeProfiles["*"] = np
+       s.testHandler.NodeProfile = &np
+
+       req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var col arvados.Collection
+       c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
+       c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
+       c.Check(col.ManifestText, check.Matches,
+               `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
+`)
+}
+
+func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
+       srv := &httpserver.Server{
+               Server: http.Server{
+                       Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+                               w.WriteHeader(404)
+                       }),
+               },
+       }
+
+       c.Assert(srv.Start(), check.IsNil)
+       defer srv.Close()
+
+       np := arvados.NodeProfile{
+               Controller: arvados.SystemServiceInstance{Listen: ":"},
+               RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
+                       TLS: false, Insecure: true}}
+       s.testHandler.Cluster.NodeProfiles["*"] = np
+       s.testHandler.NodeProfile = &np
+
+       req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var col arvados.Collection
+       c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
+       c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
+       c.Check(col.ManifestText, check.Matches,
+               `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
+`)
+}

diff --git a/lib/controller/proxy.go b/lib/controller/proxy.go
index 8c643d752e6fe4d6266ec169d4ff84c3bcbf66d1..2702c560736c6f56be3f5f9bc699f35701f02b81 100644 (file)
--- a/lib/controller/proxy.go
+++ b/lib/controller/proxy.go
@@ -72,6 +72,7 @@ func (p *proxy) Do(w http.ResponseWriter,
                Header: hdrOut,
                Body:   reqIn.Body,
        }).WithContext(ctx)
+
        resp, err := client.Do(reqOut)
        if err != nil {
                httpserver.Error(w, err.Error(), http.StatusBadGateway)