tools/keep-block-check/keep-block-check_test.go

   1 package main
   2
   3 import (
   4         "fmt"
   5         "io/ioutil"
   6         "log"
   7         "os"
   8         "regexp"
   9         "strings"
  10         "testing"
  11
  12         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  13         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  14
  15         . "gopkg.in/check.v1"
  16 )
  17
  18 // Gocheck boilerplate
  19 func Test(t *testing.T) {
  20         TestingT(t)
  21 }
  22
  23 // Gocheck boilerplate
  24 var _ = Suite(&ServerRequiredSuite{})
  25 var _ = Suite(&DoMainTestSuite{})
  26
  27 type ServerRequiredSuite struct{}
  28 type DoMainTestSuite struct{}
  29
  30 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  31         arvadostest.StartAPI()
  32 }
  33
  34 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  35         arvadostest.StopAPI()
  36         arvadostest.ResetEnv()
  37 }
  38
  39 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  40         blobSigningKey = ""
  41         keepServicesJSON = ""
  42
  43         tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file")
  44         c.Check(err, IsNil)
  45         log.SetOutput(tempfile)
  46         tempLogFileName = tempfile.Name()
  47 }
  48
  49 func (s *ServerRequiredSuite) TearDownTest(c *C) {
  50         arvadostest.StopKeep(2)
  51         os.Remove(tempLogFileName)
  52 }
  53
  54 var tempLogFileName = ""
  55 var initialArgs []string
  56 var kc *keepclient.KeepClient
  57 var keepServicesJSON, blobSigningKey string
  58
  59 func (s *DoMainTestSuite) SetUpSuite(c *C) {
  60         initialArgs = os.Args
  61 }
  62
  63 func (s *DoMainTestSuite) SetUpTest(c *C) {
  64         blobSigningKey = ""
  65         keepServicesJSON = ""
  66
  67         args := []string{"keep-block-check"}
  68         os.Args = args
  69
  70         tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file")
  71         c.Check(err, IsNil)
  72         log.SetOutput(tempfile)
  73         tempLogFileName = tempfile.Name()
  74 }
  75
  76 func (s *DoMainTestSuite) TearDownTest(c *C) {
  77         os.Remove(tempLogFileName)
  78         os.Args = initialArgs
  79 }
  80
  81 var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
  82
  83 var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0"
  84 var TestHash2 = "aaaac516f788aec4f30932ffb6395c39"
  85
  86 func setupKeepBlockCheck(c *C, enforcePermissions bool) {
  87         var config apiConfig
  88         config.APIHost = os.Getenv("ARVADOS_API_HOST")
  89         config.APIToken = arvadostest.DataManagerToken
  90         config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
  91         if enforcePermissions {
  92                 blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
  93         }
  94
  95         // Start Keep servers
  96         arvadostest.StartKeep(2, enforcePermissions)
  97
  98         // setup keepclients
  99   var err error
 100         kc, err = setupKeepClient(config, keepServicesJSON)
 101         c.Check(err, IsNil)
 102 }
 103
 104 // Setup test data
 105 var allLocators []string
 106
 107 func setupTestData(c *C) {
 108         allLocators = []string{}
 109
 110         // Put a few blocks
 111         for i := 0; i < 5; i++ {
 112                 hash, _, err := kc.PutB([]byte(fmt.Sprintf("keep-block-check-test-data-%d", i)))
 113                 c.Check(err, IsNil)
 114                 allLocators = append(allLocators, strings.Split(hash, "+A")[0])
 115         }
 116 }
 117
 118 func setupConfigFile(c *C, fileName string) string {
 119         // Setup a config file
 120         file, err := ioutil.TempFile(os.TempDir(), fileName)
 121         c.Check(err, IsNil)
 122
 123         fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n"
 124         fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n"
 125         fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
 126         fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n"
 127         fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
 128
 129         _, err = file.Write([]byte(fileContent))
 130         c.Check(err, IsNil)
 131
 132         return file.Name()
 133 }
 134
 135 func setupBlockHashFile(c *C, name string, blocks []string) string {
 136         // Setup a block hash file
 137         file, err := ioutil.TempFile(os.TempDir(), name)
 138         c.Check(err, IsNil)
 139
 140         fileContent := ""
 141         for _, hash := range blocks {
 142                 fileContent += fmt.Sprintf("%s\n", hash)
 143         }
 144         _, err = file.Write([]byte(fileContent))
 145         c.Check(err, IsNil)
 146
 147         return file.Name()
 148 }
 149
 150 func checkErrorLog(c *C, blocks []string, prefix, suffix string) {
 151         buf, _ := ioutil.ReadFile(tempLogFileName)
 152         if len(blocks) == 0 {
 153                 expected := prefix + `.*` + suffix
 154                 match, _ := regexp.MatchString(expected, string(buf))
 155                 c.Assert(match, Equals, false)
 156                 return
 157         }
 158         for _, hash := range blocks {
 159                 expected := prefix + `.*` + hash + `.*` + suffix
 160                 match, _ := regexp.MatchString(expected, string(buf))
 161                 c.Assert(match, Equals, true)
 162         }
 163 }
 164
 165 func (s *ServerRequiredSuite) TestBlockCheck(c *C) {
 166         setupKeepBlockCheck(c, false)
 167         setupTestData(c)
 168         performKeepBlockCheck(kc, blobSigningKey, "", allLocators)
 169         checkErrorLog(c, []string{}, "head", "Block not found") // no errors
 170 }
 171
 172 func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) {
 173         setupKeepBlockCheck(c, true)
 174         setupTestData(c)
 175         performKeepBlockCheck(kc, blobSigningKey, "", allLocators)
 176         checkErrorLog(c, []string{}, "head", "Block not found") // no errors
 177 }
 178
 179 func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) {
 180         setupKeepBlockCheck(c, false)
 181         setupTestData(c)
 182         performKeepBlockCheck(kc, blobSigningKey, "", []string{TestHash, TestHash2})
 183         checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found")
 184 }
 185
 186 func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C) {
 187         setupKeepBlockCheck(c, false)
 188         setupTestData(c)
 189         performKeepBlockCheck(kc, blobSigningKey, "aaa", []string{TestHash, TestHash2})
 190         checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found")
 191 }
 192
 193 func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C) {
 194         setupKeepBlockCheck(c, false)
 195         setupTestData(c)
 196         performKeepBlockCheck(kc, blobSigningKey, "999", []string{TestHash, TestHash2})
 197         checkErrorLog(c, []string{}, "head", "Block not found") // no errors
 198 }
 199
 200 // Setup block-check using keepServicesJSON with fake keepservers.
 201 // Expect error during performKeepBlockCheck due to unreachable keepservers.
 202 func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) {
 203         keepServicesJSON = testKeepServicesJSON
 204         setupKeepBlockCheck(c, false)
 205         performKeepBlockCheck(kc, blobSigningKey, "", []string{TestHash, TestHash2})
 206         checkErrorLog(c, []string{TestHash, TestHash2}, "head", "no such host")
 207 }
 208
 209 func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) {
 210         setupKeepBlockCheck(c, true)
 211         setupTestData(c)
 212         performKeepBlockCheck(kc, "badblobsigningkey", "", []string{TestHash, TestHash2})
 213         checkErrorLog(c, []string{TestHash, TestHash2}, "head", "HTTP 403")
 214 }
 215
 216 // Test keep-block-check initialization with keepServicesJSON
 217 func (s *ServerRequiredSuite) TestKeepBlockCheck_InitializeWithKeepServicesJSON(c *C) {
 218         keepServicesJSON = testKeepServicesJSON
 219         setupKeepBlockCheck(c, false)
 220         found := 0
 221         for k := range kc.LocalRoots() {
 222                 if k == "zzzzz-bi6l4-123456789012340" || k == "zzzzz-bi6l4-123456789012341" {
 223                         found++
 224                 }
 225         }
 226         c.Check(found, Equals, 2)
 227 }
 228
 229 // Test loadConfig func
 230 func (s *ServerRequiredSuite) TestLoadConfig(c *C) {
 231         // Setup config file
 232         configFile := setupConfigFile(c, "config")
 233         defer os.Remove(configFile)
 234
 235         // load configuration from the file
 236         config, blobSigningKey, err := loadConfig(configFile)
 237         c.Check(err, IsNil)
 238
 239         c.Assert(config.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
 240         c.Assert(config.APIToken, Equals, arvadostest.DataManagerToken)
 241         c.Assert(config.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
 242         c.Assert(config.ExternalClient, Equals, false)
 243         c.Assert(blobSigningKey, Equals, "abcdefg")
 244 }
 245
 246 func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) {
 247         args := []string{"-prefix", "a"}
 248         os.Args = append(os.Args, args...)
 249         err := doMain()
 250         c.Check(err, NotNil)
 251         c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true)
 252 }
 253
 254 func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) {
 255         args := []string{"-config", "no-such-file"}
 256         os.Args = append(os.Args, args...)
 257         err := doMain()
 258         c.Check(err, NotNil)
 259         c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
 260 }
 261
 262 func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) {
 263         config := setupConfigFile(c, "config")
 264         defer os.Remove(config)
 265
 266         args := []string{"-config", config}
 267         os.Args = append(os.Args, args...)
 268
 269         // Start keepservers.
 270         arvadostest.StartKeep(2, false)
 271         defer arvadostest.StopKeep(2)
 272
 273         err := doMain()
 274         c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true)
 275 }
 276
 277 func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) {
 278         config := setupConfigFile(c, "config")
 279         defer os.Remove(config)
 280
 281         args := []string{"-config", config, "-block-hash-file", "no-such-file"}
 282         os.Args = append(os.Args, args...)
 283
 284         // Start keepservers.
 285         arvadostest.StartKeep(2, false)
 286         defer arvadostest.StopKeep(2)
 287
 288         err := doMain()
 289         c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
 290 }
 291
 292 func (s *DoMainTestSuite) Test_doMain(c *C) {
 293         config := setupConfigFile(c, "config")
 294         defer os.Remove(config)
 295
 296         locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2})
 297         defer os.Remove(locatorFile)
 298
 299         // Start keepservers.
 300         arvadostest.StartKeep(2, false)
 301         defer arvadostest.StopKeep(2)
 302
 303         args := []string{"-config", config, "-block-hash-file", locatorFile}
 304         os.Args = append(os.Args, args...)
 305
 306         err := doMain()
 307         c.Check(err, IsNil)
 308         checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found")
 309 }