sdk/go/arvadosclient/arvadosclient_test.go

   1 package arvadosclient
   2
   3 import (
   4         "fmt"
   5         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   6         . "gopkg.in/check.v1"
   7         "net"
   8         "net/http"
   9         "os"
  10         "testing"
  11         "time"
  12 )
  13
  14 // Gocheck boilerplate
  15 func Test(t *testing.T) {
  16         TestingT(t)
  17 }
  18
  19 var _ = Suite(&ServerRequiredSuite{})
  20 var _ = Suite(&UnitSuite{})
  21 var _ = Suite(&MockArvadosServerSuite{})
  22
  23 // Tests that require the Keep server running
  24 type ServerRequiredSuite struct{}
  25
  26 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  27         arvadostest.StartAPI()
  28         arvadostest.StartKeep(2, false)
  29         RetryDelay = 0
  30 }
  31
  32 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  33         arvadostest.ResetEnv()
  34 }
  35
  36 func (s *ServerRequiredSuite) TestMakeArvadosClientSecure(c *C) {
  37         os.Setenv("ARVADOS_API_HOST_INSECURE", "")
  38         kc, err := MakeArvadosClient()
  39         c.Assert(err, Equals, nil)
  40         c.Check(kc.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
  41         c.Check(kc.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
  42         c.Check(kc.ApiInsecure, Equals, false)
  43 }
  44
  45 func (s *ServerRequiredSuite) TestMakeArvadosClientInsecure(c *C) {
  46         os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
  47         kc, err := MakeArvadosClient()
  48         c.Assert(err, Equals, nil)
  49         c.Check(kc.ApiInsecure, Equals, true)
  50         c.Check(kc.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
  51         c.Check(kc.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
  52         c.Check(kc.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true)
  53 }
  54
  55 func (s *ServerRequiredSuite) TestGetInvalidUUID(c *C) {
  56         arv, err := MakeArvadosClient()
  57
  58         getback := make(Dict)
  59         err = arv.Get("collections", "", nil, &getback)
  60         c.Assert(err, Equals, ErrInvalidArgument)
  61         c.Assert(len(getback), Equals, 0)
  62
  63         err = arv.Get("collections", "zebra-moose-unicorn", nil, &getback)
  64         c.Assert(err, Equals, ErrInvalidArgument)
  65         c.Assert(len(getback), Equals, 0)
  66
  67         err = arv.Get("collections", "acbd18db4cc2f85cedef654fccc4a4d8", nil, &getback)
  68         c.Assert(err, Equals, ErrInvalidArgument)
  69         c.Assert(len(getback), Equals, 0)
  70 }
  71
  72 func (s *ServerRequiredSuite) TestGetValidUUID(c *C) {
  73         arv, err := MakeArvadosClient()
  74
  75         getback := make(Dict)
  76         err = arv.Get("collections", "zzzzz-4zz18-abcdeabcdeabcde", nil, &getback)
  77         c.Assert(err, FitsTypeOf, APIServerError{})
  78         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
  79         c.Assert(len(getback), Equals, 0)
  80
  81         err = arv.Get("collections", "acbd18db4cc2f85cedef654fccc4a4d8+3", nil, &getback)
  82         c.Assert(err, FitsTypeOf, APIServerError{})
  83         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
  84         c.Assert(len(getback), Equals, 0)
  85 }
  86
  87 func (s *ServerRequiredSuite) TestInvalidResourceType(c *C) {
  88         arv, err := MakeArvadosClient()
  89
  90         getback := make(Dict)
  91         err = arv.Get("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
  92         c.Assert(err, FitsTypeOf, APIServerError{})
  93         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
  94         c.Assert(len(getback), Equals, 0)
  95
  96         err = arv.Update("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
  97         c.Assert(err, FitsTypeOf, APIServerError{})
  98         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
  99         c.Assert(len(getback), Equals, 0)
 100
 101         err = arv.List("unicorns", nil, &getback)
 102         c.Assert(err, FitsTypeOf, APIServerError{})
 103         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
 104         c.Assert(len(getback), Equals, 0)
 105 }
 106
 107 func (s *ServerRequiredSuite) TestCreatePipelineTemplate(c *C) {
 108         arv, err := MakeArvadosClient()
 109
 110         for _, idleConnections := range []bool{
 111                 false,
 112                 true,
 113         } {
 114                 if idleConnections {
 115                         arv.lastClosedIdlesAt = time.Now().Add(-time.Minute)
 116                 } else {
 117                         arv.lastClosedIdlesAt = time.Now()
 118                 }
 119
 120                 getback := make(Dict)
 121                 err = arv.Create("pipeline_templates",
 122                         Dict{"pipeline_template": Dict{
 123                                 "name": "tmp",
 124                                 "components": Dict{
 125                                         "c1": map[string]string{"script": "script1"},
 126                                         "c2": map[string]string{"script": "script2"}}}},
 127                         &getback)
 128                 c.Assert(err, Equals, nil)
 129                 c.Assert(getback["name"], Equals, "tmp")
 130                 c.Assert(getback["components"].(map[string]interface{})["c2"].(map[string]interface{})["script"], Equals, "script2")
 131
 132                 uuid := getback["uuid"].(string)
 133
 134                 getback = make(Dict)
 135                 err = arv.Get("pipeline_templates", uuid, nil, &getback)
 136                 c.Assert(err, Equals, nil)
 137                 c.Assert(getback["name"], Equals, "tmp")
 138                 c.Assert(getback["components"].(map[string]interface{})["c1"].(map[string]interface{})["script"], Equals, "script1")
 139
 140                 getback = make(Dict)
 141                 err = arv.Update("pipeline_templates", uuid,
 142                         Dict{
 143                                 "pipeline_template": Dict{"name": "tmp2"}},
 144                         &getback)
 145                 c.Assert(err, Equals, nil)
 146                 c.Assert(getback["name"], Equals, "tmp2")
 147
 148                 c.Assert(getback["uuid"].(string), Equals, uuid)
 149                 getback = make(Dict)
 150                 err = arv.Delete("pipeline_templates", uuid, nil, &getback)
 151                 c.Assert(err, Equals, nil)
 152                 c.Assert(getback["name"], Equals, "tmp2")
 153         }
 154 }
 155
 156 func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
 157         arv, _ := MakeArvadosClient()
 158
 159         getback := make(Dict)
 160
 161         {
 162                 err := arv.Create("logs",
 163                         Dict{"log": Dict{"bogus_attr": "foo"}},
 164                         &getback)
 165                 c.Assert(err, ErrorMatches, "arvados API server error: .*")
 166                 c.Assert(err, ErrorMatches, ".*unknown attribute: bogus_attr.*")
 167                 c.Assert(err, FitsTypeOf, APIServerError{})
 168                 c.Assert(err.(APIServerError).HttpStatusCode, Equals, 422)
 169         }
 170
 171         {
 172                 err := arv.Create("bogus",
 173                         Dict{"bogus": Dict{}},
 174                         &getback)
 175                 c.Assert(err, ErrorMatches, "arvados API server error: .*")
 176                 c.Assert(err, ErrorMatches, ".*Path not found.*")
 177                 c.Assert(err, FitsTypeOf, APIServerError{})
 178                 c.Assert(err.(APIServerError).HttpStatusCode, Equals, 404)
 179         }
 180 }
 181
 182 func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultCollectionReplication(c *C) {
 183         arv, err := MakeArvadosClient()
 184         value, err := arv.Discovery("defaultCollectionReplication")
 185         c.Assert(err, IsNil)
 186         c.Assert(value, NotNil)
 187 }
 188
 189 func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
 190         arv, err := MakeArvadosClient()
 191         value, err := arv.Discovery("noSuchParameter")
 192         c.Assert(err, NotNil)
 193         c.Assert(value, IsNil)
 194 }
 195
 196 type UnitSuite struct{}
 197
 198 func (s *UnitSuite) TestUUIDMatch(c *C) {
 199         c.Assert(UUIDMatch("zzzzz-tpzed-000000000000000"), Equals, true)
 200         c.Assert(UUIDMatch("zzzzz-zebra-000000000000000"), Equals, true)
 201         c.Assert(UUIDMatch("00000-00000-zzzzzzzzzzzzzzz"), Equals, true)
 202         c.Assert(UUIDMatch("ZEBRA-HORSE-AFRICANELEPHANT"), Equals, false)
 203         c.Assert(UUIDMatch(" zzzzz-tpzed-000000000000000"), Equals, false)
 204         c.Assert(UUIDMatch("d41d8cd98f00b204e9800998ecf8427e"), Equals, false)
 205         c.Assert(UUIDMatch("d41d8cd98f00b204e9800998ecf8427e+0"), Equals, false)
 206         c.Assert(UUIDMatch(""), Equals, false)
 207 }
 208
 209 func (s *UnitSuite) TestPDHMatch(c *C) {
 210         c.Assert(PDHMatch("zzzzz-tpzed-000000000000000"), Equals, false)
 211         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e"), Equals, false)
 212         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+0"), Equals, true)
 213         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+12345"), Equals, true)
 214         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e 12345"), Equals, false)
 215         c.Assert(PDHMatch("D41D8CD98F00B204E9800998ECF8427E+12345"), Equals, false)
 216         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+12345 "), Equals, false)
 217         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+abcdef"), Equals, false)
 218         c.Assert(PDHMatch("da39a3ee5e6b4b0d3255bfef95601890afd80709"), Equals, false)
 219         c.Assert(PDHMatch("da39a3ee5e6b4b0d3255bfef95601890afd80709+0"), Equals, false)
 220         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427+12345"), Equals, false)
 221         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+12345\n"), Equals, false)
 222         c.Assert(PDHMatch("+12345"), Equals, false)
 223         c.Assert(PDHMatch(""), Equals, false)
 224 }
 225
 226 // Tests that use mock arvados server
 227 type MockArvadosServerSuite struct{}
 228
 229 func (s *MockArvadosServerSuite) SetUpSuite(c *C) {
 230         RetryDelay = 0
 231 }
 232
 233 func (s *MockArvadosServerSuite) SetUpTest(c *C) {
 234         arvadostest.ResetEnv()
 235 }
 236
 237 type APIServer struct {
 238         listener net.Listener
 239         url      string
 240 }
 241
 242 func RunFakeArvadosServer(st http.Handler) (api APIServer, err error) {
 243         api.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
 244         if err != nil {
 245                 return
 246         }
 247         api.url = api.listener.Addr().String()
 248         go http.Serve(api.listener, st)
 249         return
 250 }
 251
 252 type APIStub struct {
 253         method        string
 254         retryAttempts int
 255         expected      int
 256         respStatus    []int
 257         responseBody  []string
 258 }
 259
 260 func (h *APIStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 261         if req.URL.Path == "/redirect-loop" {
 262                 http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
 263                 return
 264         }
 265         if h.respStatus[h.retryAttempts] < 0 {
 266                 // Fail the client's Do() by starting a redirect loop
 267                 http.Redirect(resp, req, "/redirect-loop", http.StatusFound)
 268         } else {
 269                 resp.WriteHeader(h.respStatus[h.retryAttempts])
 270                 resp.Write([]byte(h.responseBody[h.retryAttempts]))
 271         }
 272         h.retryAttempts++
 273 }
 274
 275 func (s *MockArvadosServerSuite) TestWithRetries(c *C) {
 276         for _, stub := range []APIStub{
 277                 {
 278                         "get", 0, 200, []int{200, 500}, []string{`{"ok":"ok"}`, ``},
 279                 },
 280                 {
 281                         "create", 0, 200, []int{200, 500}, []string{`{"ok":"ok"}`, ``},
 282                 },
 283                 {
 284                         "get", 0, 500, []int{500, 500, 500, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 285                 },
 286                 {
 287                         "create", 0, 500, []int{500, 500, 500, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 288                 },
 289                 {
 290                         "update", 0, 500, []int{500, 500, 500, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 291                 },
 292                 {
 293                         "delete", 0, 500, []int{500, 500, 500, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 294                 },
 295                 {
 296                         "get", 0, 502, []int{500, 500, 502, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 297                 },
 298                 {
 299                         "create", 0, 502, []int{500, 500, 502, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 300                 },
 301                 {
 302                         "get", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 303                 },
 304                 {
 305                         "create", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 306                 },
 307                 {
 308                         "delete", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 309                 },
 310                 {
 311                         "update", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 312                 },
 313                 {
 314                         "get", 0, 401, []int{401, 200}, []string{``, `{"ok":"ok"}`},
 315                 },
 316                 {
 317                         "create", 0, 401, []int{401, 200}, []string{``, `{"ok":"ok"}`},
 318                 },
 319                 {
 320                         "get", 0, 404, []int{404, 200}, []string{``, `{"ok":"ok"}`},
 321                 },
 322                 {
 323                         "get", 0, 401, []int{500, 401, 200}, []string{``, ``, `{"ok":"ok"}`},
 324                 },
 325
 326                 // Response code -1 simulates an HTTP/network error
 327                 // (i.e., Do() returns an error; there is no HTTP
 328                 // response status code).
 329
 330                 // Succeed on second retry
 331                 {
 332                         "get", 0, 200, []int{-1, -1, 200}, []string{``, ``, `{"ok":"ok"}`},
 333                 },
 334                 // "POST" is not safe to retry: fail after one error
 335                 {
 336                         "create", 0, -1, []int{-1, 200}, []string{``, `{"ok":"ok"}`},
 337                 },
 338         } {
 339                 api, err := RunFakeArvadosServer(&stub)
 340                 c.Check(err, IsNil)
 341
 342                 defer api.listener.Close()
 343
 344                 arv := ArvadosClient{
 345                         Scheme:      "http",
 346                         ApiServer:   api.url,
 347                         ApiToken:    "abc123",
 348                         ApiInsecure: true,
 349                         Client:      &http.Client{Transport: &http.Transport{}},
 350                         Retries:     2}
 351
 352                 getback := make(Dict)
 353                 switch stub.method {
 354                 case "get":
 355                         err = arv.Get("collections", "zzzzz-4zz18-znfnqtbbv4spc3w", nil, &getback)
 356                 case "create":
 357                         err = arv.Create("collections",
 358                                 Dict{"collection": Dict{"name": "testing"}},
 359                                 &getback)
 360                 case "update":
 361                         err = arv.Update("collections", "zzzzz-4zz18-znfnqtbbv4spc3w",
 362                                 Dict{"collection": Dict{"name": "testing"}},
 363                                 &getback)
 364                 case "delete":
 365                         err = arv.Delete("pipeline_templates", "zzzzz-4zz18-znfnqtbbv4spc3w", nil, &getback)
 366                 }
 367
 368                 switch stub.expected {
 369                 case 200:
 370                         c.Check(err, IsNil)
 371                         c.Check(getback["ok"], Equals, "ok")
 372                 case -1:
 373                         c.Check(err, NotNil)
 374                         c.Check(err, ErrorMatches, `.*stopped after \d+ redirects`)
 375                 default:
 376                         c.Check(err, NotNil)
 377                         c.Check(err, ErrorMatches, fmt.Sprintf("arvados API server error: %d.*", stub.expected))
 378                         c.Check(err.(APIServerError).HttpStatusCode, Equals, stub.expected)
 379                 }
 380         }
 381 }