sdk/go/arvadosclient/arvadosclient_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvadosclient
   6
   7 import (
   8         "fmt"
   9         "net"
  10         "net/http"
  11         "os"
  12         "testing"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/arvados"
  16         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  17         . "gopkg.in/check.v1"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 var _ = Suite(&ServerRequiredSuite{})
  26 var _ = Suite(&UnitSuite{})
  27 var _ = Suite(&MockArvadosServerSuite{})
  28
  29 // Tests that require the Keep server running
  30 type ServerRequiredSuite struct{}
  31
  32 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  33         arvadostest.StartKeep(2, false)
  34         RetryDelay = 2 * time.Second
  35 }
  36
  37 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  38         arvadostest.StopKeep(2)
  39 }
  40
  41 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  42         arvadostest.ResetEnv()
  43 }
  44
  45 func (s *ServerRequiredSuite) TestMakeArvadosClientSecure(c *C) {
  46         os.Setenv("ARVADOS_API_HOST_INSECURE", "")
  47         ac, err := MakeArvadosClient()
  48         c.Assert(err, Equals, nil)
  49         c.Check(ac.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
  50         c.Check(ac.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
  51         c.Check(ac.ApiInsecure, Equals, false)
  52 }
  53
  54 func (s *ServerRequiredSuite) TestMakeArvadosClientInsecure(c *C) {
  55         os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
  56         ac, err := MakeArvadosClient()
  57         c.Assert(err, Equals, nil)
  58         c.Check(ac.ApiInsecure, Equals, true)
  59         c.Check(ac.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
  60         c.Check(ac.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
  61         c.Check(ac.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true)
  62 }
  63
  64 func (s *ServerRequiredSuite) TestGetInvalidUUID(c *C) {
  65         arv, err := MakeArvadosClient()
  66         c.Assert(err, IsNil)
  67
  68         getback := make(Dict)
  69         err = arv.Get("collections", "", nil, &getback)
  70         c.Assert(err, Equals, ErrInvalidArgument)
  71         c.Assert(len(getback), Equals, 0)
  72
  73         err = arv.Get("collections", "zebra-moose-unicorn", nil, &getback)
  74         c.Assert(err, Equals, ErrInvalidArgument)
  75         c.Assert(len(getback), Equals, 0)
  76
  77         err = arv.Get("collections", "acbd18db4cc2f85cedef654fccc4a4d8", nil, &getback)
  78         c.Assert(err, Equals, ErrInvalidArgument)
  79         c.Assert(len(getback), Equals, 0)
  80 }
  81
  82 func (s *ServerRequiredSuite) TestGetValidUUID(c *C) {
  83         arv, err := MakeArvadosClient()
  84         c.Assert(err, IsNil)
  85
  86         getback := make(Dict)
  87         err = arv.Get("collections", "zzzzz-4zz18-abcdeabcdeabcde", nil, &getback)
  88         c.Assert(err, FitsTypeOf, APIServerError{})
  89         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
  90         c.Assert(len(getback), Equals, 0)
  91
  92         err = arv.Get("collections", "acbd18db4cc2f85cedef654fccc4a4d8+3", nil, &getback)
  93         c.Assert(err, FitsTypeOf, APIServerError{})
  94         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
  95         c.Assert(len(getback), Equals, 0)
  96 }
  97
  98 func (s *ServerRequiredSuite) TestInvalidResourceType(c *C) {
  99         arv, err := MakeArvadosClient()
 100         c.Assert(err, IsNil)
 101
 102         getback := make(Dict)
 103         err = arv.Get("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
 104         c.Assert(err, FitsTypeOf, APIServerError{})
 105         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
 106         c.Assert(len(getback), Equals, 0)
 107
 108         err = arv.Update("unicorns", "zzzzz-zebra-unicorn7unicorn", nil, &getback)
 109         c.Assert(err, FitsTypeOf, APIServerError{})
 110         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
 111         c.Assert(len(getback), Equals, 0)
 112
 113         err = arv.List("unicorns", nil, &getback)
 114         c.Assert(err, FitsTypeOf, APIServerError{})
 115         c.Assert(err.(APIServerError).HttpStatusCode, Equals, http.StatusNotFound)
 116         c.Assert(len(getback), Equals, 0)
 117 }
 118
 119 func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
 120         arv, _ := MakeArvadosClient()
 121
 122         getback := make(Dict)
 123
 124         {
 125                 err := arv.Create("logs",
 126                         Dict{"log": Dict{"bogus_attr": "foo"}},
 127                         &getback)
 128                 c.Assert(err, ErrorMatches, "arvados API server error: .*")
 129                 c.Assert(err, ErrorMatches, ".*unknown attribute(: | ')bogus_attr.*")
 130                 c.Assert(err, FitsTypeOf, APIServerError{})
 131                 c.Assert(err.(APIServerError).HttpStatusCode, Equals, 422)
 132         }
 133
 134         {
 135                 err := arv.Create("bogus",
 136                         Dict{"bogus": Dict{}},
 137                         &getback)
 138                 c.Assert(err, ErrorMatches, "arvados API server error: .*")
 139                 c.Assert(err, ErrorMatches, ".*Path not found.*")
 140                 c.Assert(err, FitsTypeOf, APIServerError{})
 141                 c.Assert(err.(APIServerError).HttpStatusCode, Equals, 404)
 142         }
 143 }
 144
 145 func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultCollectionReplication(c *C) {
 146         arv, err := MakeArvadosClient()
 147         c.Assert(err, IsNil)
 148         value, err := arv.Discovery("defaultCollectionReplication")
 149         c.Assert(err, IsNil)
 150         c.Assert(value, NotNil)
 151 }
 152
 153 func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
 154         arv, err := MakeArvadosClient()
 155         c.Assert(err, IsNil)
 156         value, err := arv.Discovery("noSuchParameter")
 157         c.Assert(err, NotNil)
 158         c.Assert(value, IsNil)
 159 }
 160
 161 func (s *ServerRequiredSuite) TestAPIClusterConfig_Get_StorageClasses(c *C) {
 162         arv, err := MakeArvadosClient()
 163         c.Assert(err, IsNil)
 164         data, err := arv.ClusterConfig("StorageClasses")
 165         c.Assert(err, IsNil)
 166         c.Assert(data, NotNil)
 167         clusterConfig := data.(map[string]interface{})
 168         _, ok := clusterConfig["default"]
 169         c.Assert(ok, Equals, true)
 170 }
 171
 172 func (s *ServerRequiredSuite) TestAPIClusterConfig_Get_All(c *C) {
 173         arv, err := MakeArvadosClient()
 174         c.Assert(err, IsNil)
 175         data, err := arv.ClusterConfig("")
 176         c.Assert(err, IsNil)
 177         c.Assert(data, NotNil)
 178         clusterConfig := data.(map[string]interface{})
 179         _, ok := clusterConfig["StorageClasses"]
 180         c.Assert(ok, Equals, true)
 181 }
 182
 183 func (s *ServerRequiredSuite) TestAPIClusterConfig_Get_noSuchSection(c *C) {
 184         arv, err := MakeArvadosClient()
 185         c.Assert(err, IsNil)
 186         data, err := arv.ClusterConfig("noSuchSection")
 187         c.Assert(err, NotNil)
 188         c.Assert(data, IsNil)
 189 }
 190
 191 func (s *ServerRequiredSuite) TestCreateLarge(c *C) {
 192         arv, err := MakeArvadosClient()
 193         c.Assert(err, IsNil)
 194
 195         txt := arvados.SignLocator("d41d8cd98f00b204e9800998ecf8427e+0", arv.ApiToken, time.Now().Add(time.Minute), time.Minute, []byte(arvadostest.SystemRootToken))
 196         // Ensure our request body is bigger than the Go http server's
 197         // default max size, 10 MB.
 198         for len(txt) < 12000000 {
 199                 txt = txt + " " + txt
 200         }
 201         txt = ". " + txt + " 0:0:foo\n"
 202
 203         resp := Dict{}
 204         err = arv.Create("collections", Dict{
 205                 "ensure_unique_name": true,
 206                 "collection": Dict{
 207                         "is_trashed":    true,
 208                         "name":          "test",
 209                         "manifest_text": txt,
 210                 },
 211         }, &resp)
 212         c.Check(err, IsNil)
 213         c.Check(resp["portable_data_hash"], Not(Equals), "")
 214         c.Check(resp["portable_data_hash"], Not(Equals), "d41d8cd98f00b204e9800998ecf8427e+0")
 215 }
 216
 217 type UnitSuite struct{}
 218
 219 func (s *UnitSuite) TestUUIDMatch(c *C) {
 220         c.Assert(UUIDMatch("zzzzz-tpzed-000000000000000"), Equals, true)
 221         c.Assert(UUIDMatch("zzzzz-zebra-000000000000000"), Equals, true)
 222         c.Assert(UUIDMatch("00000-00000-zzzzzzzzzzzzzzz"), Equals, true)
 223         c.Assert(UUIDMatch("ZEBRA-HORSE-AFRICANELEPHANT"), Equals, false)
 224         c.Assert(UUIDMatch(" zzzzz-tpzed-000000000000000"), Equals, false)
 225         c.Assert(UUIDMatch("d41d8cd98f00b204e9800998ecf8427e"), Equals, false)
 226         c.Assert(UUIDMatch("d41d8cd98f00b204e9800998ecf8427e+0"), Equals, false)
 227         c.Assert(UUIDMatch(""), Equals, false)
 228 }
 229
 230 func (s *UnitSuite) TestPDHMatch(c *C) {
 231         c.Assert(PDHMatch("zzzzz-tpzed-000000000000000"), Equals, false)
 232         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e"), Equals, false)
 233         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+0"), Equals, true)
 234         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+12345"), Equals, true)
 235         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e 12345"), Equals, false)
 236         c.Assert(PDHMatch("D41D8CD98F00B204E9800998ECF8427E+12345"), Equals, false)
 237         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+12345 "), Equals, false)
 238         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+abcdef"), Equals, false)
 239         c.Assert(PDHMatch("da39a3ee5e6b4b0d3255bfef95601890afd80709"), Equals, false)
 240         c.Assert(PDHMatch("da39a3ee5e6b4b0d3255bfef95601890afd80709+0"), Equals, false)
 241         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427+12345"), Equals, false)
 242         c.Assert(PDHMatch("d41d8cd98f00b204e9800998ecf8427e+12345\n"), Equals, false)
 243         c.Assert(PDHMatch("+12345"), Equals, false)
 244         c.Assert(PDHMatch(""), Equals, false)
 245 }
 246
 247 // Tests that use mock arvados server
 248 type MockArvadosServerSuite struct{}
 249
 250 func (s *MockArvadosServerSuite) SetUpSuite(c *C) {
 251         RetryDelay = 100 * time.Millisecond
 252 }
 253
 254 func (s *MockArvadosServerSuite) SetUpTest(c *C) {
 255         arvadostest.ResetEnv()
 256 }
 257
 258 type APIServer struct {
 259         listener net.Listener
 260         url      string
 261 }
 262
 263 func RunFakeArvadosServer(st http.Handler) (api APIServer, err error) {
 264         api.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
 265         if err != nil {
 266                 return
 267         }
 268         api.url = api.listener.Addr().String()
 269         go http.Serve(api.listener, st)
 270         return
 271 }
 272
 273 type APIStub struct {
 274         method        string
 275         retryAttempts int
 276         expected      int
 277         respStatus    []int
 278         responseBody  []string
 279 }
 280
 281 func (h *APIStub) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 282         if status := h.respStatus[h.retryAttempts]; status < 0 {
 283                 // Fail the client's Do() by hanging up without
 284                 // sending an HTTP response header.
 285                 conn, _, err := resp.(http.Hijacker).Hijack()
 286                 if err != nil {
 287                         panic(err)
 288                 }
 289                 conn.Write([]byte("zzzzzzzzzz"))
 290                 conn.Close()
 291         } else {
 292                 resp.WriteHeader(status)
 293                 resp.Write([]byte(h.responseBody[h.retryAttempts]))
 294         }
 295         h.retryAttempts++
 296 }
 297
 298 func (s *MockArvadosServerSuite) TestWithRetries(c *C) {
 299         for _, stub := range []APIStub{
 300                 {
 301                         "get", 0, 200, []int{200, 500}, []string{`{"ok":"ok"}`, ``},
 302                 },
 303                 {
 304                         "create", 0, 200, []int{200, 500}, []string{`{"ok":"ok"}`, ``},
 305                 },
 306                 {
 307                         "get", 0, 423, []int{500, 500, 423, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 308                 },
 309                 {
 310                         "create", 0, 423, []int{500, 500, 423, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 311                 },
 312                 {
 313                         "update", 0, 422, []int{500, 500, 422, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 314                 },
 315                 {
 316                         "delete", 0, 422, []int{500, 500, 422, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 317                 },
 318                 {
 319                         "get", 0, 401, []int{500, 502, 401, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 320                 },
 321                 {
 322                         "create", 0, 422, []int{500, 502, 422, 200}, []string{``, ``, ``, `{"ok":"ok"}`},
 323                 },
 324                 {
 325                         "get", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 326                 },
 327                 {
 328                         "create", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 329                 },
 330                 {
 331                         "delete", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 332                 },
 333                 {
 334                         "update", 0, 200, []int{500, 500, 200}, []string{``, ``, `{"ok":"ok"}`},
 335                 },
 336                 {
 337                         "get", 0, 401, []int{401, 200}, []string{``, `{"ok":"ok"}`},
 338                 },
 339                 {
 340                         "create", 0, 401, []int{401, 200}, []string{``, `{"ok":"ok"}`},
 341                 },
 342                 {
 343                         "create", 0, 403, []int{403, 200}, []string{``, `{"ok":"ok"}`},
 344                 },
 345                 {
 346                         "create", 0, 422, []int{422, 200}, []string{``, `{"ok":"ok"}`},
 347                 },
 348                 {
 349                         "get", 0, 404, []int{404, 200}, []string{``, `{"ok":"ok"}`},
 350                 },
 351                 {
 352                         "get", 0, 401, []int{500, 401, 200}, []string{``, ``, `{"ok":"ok"}`},
 353                 },
 354
 355                 // Response code -1 simulates an HTTP/network error
 356                 // (i.e., Do() returns an error; there is no HTTP
 357                 // response status code).
 358
 359                 // Succeed on second retry
 360                 {
 361                         "get", 0, 200, []int{-1, -1, 200}, []string{``, ``, `{"ok":"ok"}`},
 362                 },
 363                 // "POST" protocol error is safe to retry
 364                 {
 365                         "create", 0, 200, []int{-1, 200}, []string{``, `{"ok":"ok"}`},
 366                 },
 367         } {
 368                 c.Logf("stub: %#v", stub)
 369
 370                 api, err := RunFakeArvadosServer(&stub)
 371                 c.Check(err, IsNil)
 372
 373                 defer api.listener.Close()
 374
 375                 arv := ArvadosClient{
 376                         Scheme:      "http",
 377                         ApiServer:   api.url,
 378                         ApiToken:    "abc123",
 379                         ApiInsecure: true,
 380                         Client:      &http.Client{Transport: &http.Transport{}},
 381                         Retries:     2}
 382
 383                 getback := make(Dict)
 384                 switch stub.method {
 385                 case "get":
 386                         err = arv.Get("collections", "zzzzz-4zz18-znfnqtbbv4spc3w", nil, &getback)
 387                 case "create":
 388                         err = arv.Create("collections",
 389                                 Dict{"collection": Dict{"name": "testing"}},
 390                                 &getback)
 391                 case "update":
 392                         err = arv.Update("collections", "zzzzz-4zz18-znfnqtbbv4spc3w",
 393                                 Dict{"collection": Dict{"name": "testing"}},
 394                                 &getback)
 395                 case "delete":
 396                         err = arv.Delete("pipeline_templates", "zzzzz-4zz18-znfnqtbbv4spc3w", nil, &getback)
 397                 }
 398
 399                 switch stub.expected {
 400                 case 200:
 401                         c.Check(err, IsNil)
 402                         c.Check(getback["ok"], Equals, "ok")
 403                 case -1:
 404                         c.Check(err, NotNil)
 405                         c.Check(err, ErrorMatches, `.*stopped after \d+ redirects`)
 406                 default:
 407                         c.Check(err, NotNil)
 408                         c.Check(err, ErrorMatches, fmt.Sprintf("arvados API server error: %d.*", stub.expected))
 409                         if c.Check(err, FitsTypeOf, APIServerError{}) {
 410                                 c.Check(err.(APIServerError).HttpStatusCode, Equals, stub.expected)
 411                         }
 412                 }
 413         }
 414 }