X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/27b9247e0c42700e0c7d94f7721e2164610b1e4c..8d3aec7a9485d0bec6b672d0b9097e02a1ceea09:/sdk/go/arvadosclient/arvadosclient_test.go

diff --git a/sdk/go/arvadosclient/arvadosclient_test.go b/sdk/go/arvadosclient/arvadosclient_test.go
index a9c2c83cf6..b074e21e81 100644
--- a/sdk/go/arvadosclient/arvadosclient_test.go
+++ b/sdk/go/arvadosclient/arvadosclient_test.go
@@ -1,16 +1,20 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package arvadosclient
 
 import (
 	"fmt"
-	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
-	. "gopkg.in/check.v1"
 	"net"
 	"net/http"
 	"os"
-	"strconv"
-	"strings"
 	"testing"
 	"time"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	. "gopkg.in/check.v1"
 )
 
 // Gocheck boilerplate
@@ -26,9 +30,12 @@ var _ = Suite(&MockArvadosServerSuite{})
 type ServerRequiredSuite struct{}
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	arvadostest.StartAPI()
 	arvadostest.StartKeep(2, false)
-	RetryDelay = 0
+	RetryDelay = 2 * time.Second
+}
+
+func (s *ServerRequiredSuite) TearDownSuite(c *C) {
+	arvadostest.StopKeep(2)
 }
 
 func (s *ServerRequiredSuite) SetUpTest(c *C) {
@@ -37,25 +44,26 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) {
 
 func (s *ServerRequiredSuite) TestMakeArvadosClientSecure(c *C) {
 	os.Setenv("ARVADOS_API_HOST_INSECURE", "")
-	kc, err := MakeArvadosClient()
+	ac, err := MakeArvadosClient()
 	c.Assert(err, Equals, nil)
-	c.Check(kc.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
-	c.Check(kc.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
-	c.Check(kc.ApiInsecure, Equals, false)
+	c.Check(ac.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
+	c.Check(ac.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
+	c.Check(ac.ApiInsecure, Equals, false)
 }
 
 func (s *ServerRequiredSuite) TestMakeArvadosClientInsecure(c *C) {
 	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-	kc, err := MakeArvadosClient()
+	ac, err := MakeArvadosClient()
 	c.Assert(err, Equals, nil)
-	c.Check(kc.ApiInsecure, Equals, true)
-	c.Check(kc.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
-	c.Check(kc.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
-	c.Check(kc.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true)
+	c.Check(ac.ApiInsecure, Equals, true)
+	c.Check(ac.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
+	c.Check(ac.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
+	c.Check(ac.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true)
 }
 
 func (s *ServerRequiredSuite) TestGetInvalidUUID(c *C) {
 	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
 
 	getback := make(Dict)
 	err = arv.Get("collections", "", nil, &getback)
@@ -73,6 +81,7 @@ func (s *ServerRequiredSuite) TestGetInvalidUUID(c *C) {
 
 func (s *ServerRequiredSuite) TestGetValidUUID(c *C) {
 	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
 
 	getback := make(Dict)
 	err = arv.Get("collections", "zzzzz-4zz18-abcdeabcdeabcde", nil, &getback)
@@ -88,6 +97,7 @@ func (s *ServerRequiredSuite) TestGetValidUUID(c *C) {
 
 func (s *ServerRequiredSuite) TestInvalidResourceType(c *C) {
 	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
 
 	getback := make(Dict)
 	err = arv.Get("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
@@ -106,55 +116,6 @@ func (s *ServerRequiredSuite) TestInvalidResourceType(c *C) {
 	c.Assert(len(getback), Equals, 0)
 }
 
-func (s *ServerRequiredSuite) TestCreatePipelineTemplate(c *C) {
-	arv, err := MakeArvadosClient()
-
-	for _, idleConnections := range []bool{
-		false,
-		true,
-	} {
-		if idleConnections {
-			arv.lastClosedIdlesAt = time.Now().Add(-time.Minute)
-		} else {
-			arv.lastClosedIdlesAt = time.Now()
-		}
-
-		getback := make(Dict)
-		err = arv.Create("pipeline_templates",
-			Dict{"pipeline_template": Dict{
-				"name": "tmp",
-				"components": Dict{
-					"c1": map[string]string{"script": "script1"},
-					"c2": map[string]string{"script": "script2"}}}},
-			&getback)
-		c.Assert(err, Equals, nil)
-		c.Assert(getback["name"], Equals, "tmp")
-		c.Assert(getback["components"].(map[string]interface{})["c2"].(map[string]interface{})["script"], Equals, "script2")
-
-		uuid := getback["uuid"].(string)
-
-		getback = make(Dict)
-		err = arv.Get("pipeline_templates", uuid, nil, &getback)
-		c.Assert(err, Equals, nil)
-		c.Assert(getback["name"], Equals, "tmp")
-		c.Assert(getback["components"].(map[string]interface{})["c1"].(map[string]interface{})["script"], Equals, "script1")
-
-		getback = make(Dict)
-		err = arv.Update("pipeline_templates", uuid,
-			Dict{
-				"pipeline_template": Dict{"name": "tmp2"}},
-			&getback)
-		c.Assert(err, Equals, nil)
-		c.Assert(getback["name"], Equals, "tmp2")
-
-		c.Assert(getback["uuid"].(string), Equals, uuid)
-		getback = make(Dict)
-		err = arv.Delete("pipeline_templates", uuid, nil, &getback)
-		c.Assert(err, Equals, nil)
-		c.Assert(getback["name"], Equals, "tmp2")
-	}
-}
-
 func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
 	arv, _ := MakeArvadosClient()
 
@@ -165,7 +126,7 @@ func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
 			Dict{"log": Dict{"bogus_attr": "foo"}},
 			&getback)
 		c.Assert(err, ErrorMatches, "arvados API server error: .*")
-		c.Assert(err, ErrorMatches, ".*unknown attribute: bogus_attr.*")
+		c.Assert(err, ErrorMatches, ".*unknown attribute(: | ')bogus_attr.*")
 		c.Assert(err, FitsTypeOf, APIServerError{})
 		c.Assert(err.(APIServerError).HttpStatusCode, Equals, 422)
 	}
@@ -183,6 +144,7 @@ func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
 
 func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultCollectionReplication(c *C) {
 	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
 	value, err := arv.Discovery("defaultCollectionReplication")
 	c.Assert(err, IsNil)
 	c.Assert(value, NotNil)
@@ -190,11 +152,68 @@ func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultCollectionReplication(
 
 func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
 	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
 	value, err := arv.Discovery("noSuchParameter")
 	c.Assert(err, NotNil)
 	c.Assert(value, IsNil)
 }
 
+func (s *ServerRequiredSuite) TestAPIClusterConfig_Get_StorageClasses(c *C) {
+	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
+	data, err := arv.ClusterConfig("StorageClasses")
+	c.Assert(err, IsNil)
+	c.Assert(data, NotNil)
+	clusterConfig := data.(map[string]interface{})
+	_, ok := clusterConfig["default"]
+	c.Assert(ok, Equals, true)
+}
+
+func (s *ServerRequiredSuite) TestAPIClusterConfig_Get_All(c *C) {
+	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
+	data, err := arv.ClusterConfig("")
+	c.Assert(err, IsNil)
+	c.Assert(data, NotNil)
+	clusterConfig := data.(map[string]interface{})
+	_, ok := clusterConfig["StorageClasses"]
+	c.Assert(ok, Equals, true)
+}
+
+func (s *ServerRequiredSuite) TestAPIClusterConfig_Get_noSuchSection(c *C) {
+	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
+	data, err := arv.ClusterConfig("noSuchSection")
+	c.Assert(err, NotNil)
+	c.Assert(data, IsNil)
+}
+
+func (s *ServerRequiredSuite) TestCreateLarge(c *C) {
+	arv, err := MakeArvadosClient()
+	c.Assert(err, IsNil)
+
+	txt := arvados.SignLocator("d41d8cd98f00b204e9800998ecf8427e+0", arv.ApiToken, time.Now().Add(time.Minute), time.Minute, []byte(arvadostest.SystemRootToken))
+	// Ensure our request body is bigger than the Go http server's
+	// default max size, 10 MB.
+	for len(txt) < 12000000 {
+		txt = txt + " " + txt
+	}
+	txt = ". " + txt + " 0:0:foo\n"
+
+	resp := Dict{}
+	err = arv.Create("collections", Dict{
+		"ensure_unique_name": true,
+		"collection": Dict{
+			"is_trashed":    true,
+			"name":          "test",
+			"manifest_text": txt,
+		},
+	}, &resp)
+	c.Check(err, IsNil)
+	c.Check(resp["portable_data_hash"], Not(Equals), "")
+	c.Check(resp["portable_data_hash"], Not(Equals), "d41d8cd98f00b204e9800998ecf8427e+0")
+}
+
 type UnitSuite struct{}
 
 func (s *UnitSuite) TestUUIDMatch(c *C) {
@@ -229,7 +248,7 @@ func (s *UnitSuite) TestPDHMatch(c *C) {
 type MockArvadosServerSuite struct{}
 
 func (s *MockArvadosServerSuite) SetUpSuite(c *C) {
-	RetryDelay = 0
+	RetryDelay = 100 * time.Millisecond
 }
 
 func (s *MockArvadosServerSuite) SetUpTest(c *C) {
@@ -252,59 +271,103 @@ func RunFakeArvadosServer(st http.Handler) (api APIServer, err error) {
 }
 
 type APIStub struct {
-	count      int
-	respStatus []int
+	method        string
+	retryAttempts int
+	expected      int
+	respStatus    []int
+	responseBody  []string
 }
 
 func (h *APIStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	resp.WriteHeader(h.respStatus[h.count])
-
-	if h.respStatus[h.count] == 200 {
-		resp.Write([]byte(`{"ok":"ok"}`))
+	if status := h.respStatus[h.retryAttempts]; status < 0 {
+		// Fail the client's Do() by hanging up without
+		// sending an HTTP response header.
+		conn, _, err := resp.(http.Hijacker).Hijack()
+		if err != nil {
+			panic(err)
+		}
+		conn.Write([]byte("zzzzzzzzzz"))
+		conn.Close()
 	} else {
-		resp.Write([]byte(``))
+		resp.WriteHeader(status)
+		resp.Write([]byte(h.responseBody[h.retryAttempts]))
 	}
-
-	h.count += 1
+	h.retryAttempts++
 }
 
 func (s *MockArvadosServerSuite) TestWithRetries(c *C) {
-	// Each testCase below specifies the operation to be used ("get", "create" etc),
-	// the "expected" outcome (500 or 401 or success etc,
-	// and an array of response statuses to be returned in that order for each (re)try.
-	//
-	// The tests are using retry count of 2,
-	// and hence the first "non-retryable" code (such as 401)
-	// or whatever is the third status code is to be expected.
-	for _, testCase := range []map[string][]int{
-		{"get:500": []int{500, 500, 500, 200}},
-		{"create:500": []int{500, 500, 500, 200}},
-		{"update:500": []int{500, 500, 500, 200}},
-		{"delete:500": []int{500, 500, 500, 200}},
-		{"get:502": []int{500, 500, 502, 200}},
-		{"create:502": []int{500, 500, 502, 200}},
-		{"get:success": []int{500, 500, 200}},
-		{"create:success": []int{500, 500, 200}},
-		{"get:401": []int{401, 200}},
-		{"create:401": []int{401, 200}},
-		{"get:404": []int{404, 200}},
-		{"create:404": []int{404, 200}},
-		{"get:401": []int{500, 401, 200}},
-		{"create:401": []int{500, 401, 200}},
+	for _, stub := range []APIStub{
+		{
+			"get", 0, 200, []int{200, 500}, []string{`{"ok":"ok"}`, ``},
+		},
+		{
+			"create", 0, 200, []int{200, 500}, []string{`{"ok":"ok"}`, ``},
+		},
+		{
+			"get", 0, 423, []int{500, 500, 423, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"create", 0, 423, []int{500, 500, 423, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"update", 0, 422, []int{500, 500, 422, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"delete", 0, 422, []int{500, 500, 422, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"get", 0, 401, []int{500, 502, 401, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"create", 0, 422, []int{500, 502, 422, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"get", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"create", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"delete", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"update", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
+		},
+		{
+			"get", 0, 401, []int{401, 200}, []string{``, `{"ok":"ok"}`},
+		},
+		{
+			"create", 0, 401, []int{401, 200}, []string{``, `{"ok":"ok"}`},
+		},
+		{
+			"create", 0, 403, []int{403, 200}, []string{``, `{"ok":"ok"}`},
+		},
+		{
+			"create", 0, 422, []int{422, 200}, []string{``, `{"ok":"ok"}`},
+		},
+		{
+			"get", 0, 404, []int{404, 200}, []string{``, `{"ok":"ok"}`},
+		},
+		{
+			"get", 0, 401, []int{500, 401, 200}, []string{``, ``, `{"ok":"ok"}`},
+		},
+
+		// Response code -1 simulates an HTTP/network error
+		// (i.e., Do() returns an error; there is no HTTP
+		// response status code).
+
+		// Succeed on second retry
+		{
+			"get", 0, 200, []int{-1, -1, 200}, []string{``, ``, `{"ok":"ok"}`},
+		},
+		// "POST" protocol error is safe to retry
+		{
+			"create", 0, 200, []int{-1, 200}, []string{``, `{"ok":"ok"}`},
+		},
 	} {
-		var method string
-		var statusCodes []int
-		var expected string
-
-		for key, value := range testCase {
-			method = key[:strings.Index(key, ":")]
-			expected = key[strings.Index(key, ":")+1:]
-			statusCodes = value
-		}
-
-		stub := &APIStub{0, statusCodes}
+		c.Logf("stub: %#v", stub)
 
-		api, err := RunFakeArvadosServer(stub)
+		api, err := RunFakeArvadosServer(&stub)
 		c.Check(err, IsNil)
 
 		defer api.listener.Close()
@@ -318,7 +381,7 @@ func (s *MockArvadosServerSuite) TestWithRetries(c *C) {
 			Retries:     2}
 
 		getback := make(Dict)
-		switch method {
+		switch stub.method {
 		case "get":
 			err = arv.Get("collections", "zzzzz-4zz18-znfnqtbbv4spc3w", nil, &getback)
 		case "create":
@@ -333,14 +396,19 @@ func (s *MockArvadosServerSuite) TestWithRetries(c *C) {
 			err = arv.Delete("pipeline_templates", "zzzzz-4zz18-znfnqtbbv4spc3w", nil, &getback)
 		}
 
-		if expected == "success" {
+		switch stub.expected {
+		case 200:
 			c.Check(err, IsNil)
-			c.Assert(getback["ok"], Equals, "ok")
-		} else {
+			c.Check(getback["ok"], Equals, "ok")
+		case -1:
+			c.Check(err, NotNil)
+			c.Check(err, ErrorMatches, `.*stopped after \d+ redirects`)
+		default:
 			c.Check(err, NotNil)
-			expectedStatus, _ := strconv.Atoi(expected)
-			c.Check(strings.Contains(err.Error(), fmt.Sprintf("%s%s", "arvados API server error: ", expected)), Equals, true)
-			c.Assert(err.(APIServerError).HttpStatusCode, Equals, expectedStatus)
+			c.Check(err, ErrorMatches, fmt.Sprintf("arvados API server error: %d.*", stub.expected))
+			if c.Check(err, FitsTypeOf, APIServerError{}) {
+				c.Check(err.(APIServerError).HttpStatusCode, Equals, stub.expected)
+			}
 		}
 	}
 }