From: Tom Clegg Date: Wed, 9 Sep 2015 19:08:09 +0000 (-0400) Subject: 6260: Just use the token loaded by the SDK (ARVADOS_API_TOKEN), X-Git-Tag: 1.1.0~1354^2~15 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/b4866c16547c6c3196d5d8327686986e8997c187 6260: Just use the token loaded by the SDK (ARVADOS_API_TOKEN), instead of requiring a token separately on the command line. --- diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go index 9cfd2ee5d7..3fb135cb3a 100644 --- a/services/datamanager/datamanager.go +++ b/services/datamanager/datamanager.go @@ -68,8 +68,6 @@ func makeArvadosClient() arvadosclient.ArvadosClient { return arv } -var dataManagerToken string - func singlerun(arv arvadosclient.ArvadosClient) error { var err error if is_admin, err := util.UserIsAdmin(arv); err != nil { @@ -93,21 +91,6 @@ func singlerun(arv arvadosclient.ArvadosClient) error { arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc) } - // Verify that datamanager token belongs to an admin user - if dataManagerToken == "" { - dataManagerToken = keep.GetDataManagerToken(arvLogger) - } - origArvToken := arv.ApiToken - arv.ApiToken = dataManagerToken - if is_admin, err := util.UserIsAdmin(arv); err != nil { - log.Printf("Error querying arvados user for data manager token %s", err.Error()) - return err - } else if !is_admin { - log.Printf("Datamanager token does not belong to an admin user.") - return errors.New("Datamanager token does not belong to an admin user.") - } - arv.ApiToken = origArvToken - var ( dataFetcher summary.DataFetcher readCollections collection.ReadCollections @@ -178,7 +161,7 @@ func singlerun(arv arvadosclient.ArvadosClient) error { if trashErr != nil { return err } else { - keep.SendTrashLists(dataManagerToken, kc, trashLists) + keep.SendTrashLists(kc, trashLists) } return nil diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go index 2ab1a48159..18b302a5ba 100644 --- a/services/datamanager/datamanager_test.go +++ b/services/datamanager/datamanager_test.go @@ -6,7 +6,6 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/arvadostest" "git.curoverse.com/arvados.git/sdk/go/keepclient" - "git.curoverse.com/arvados.git/services/datamanager/keep" "io/ioutil" "net/http" "os" @@ -17,7 +16,10 @@ import ( "time" ) -const ACTIVE_USER_TOKEN = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi" +const ( + ActiveUserToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi" + AdminToken = "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h" +) var arv arvadosclient.ArvadosClient var keepClient *keepclient.KeepClient @@ -172,7 +174,7 @@ func getBlockIndexesForServer(t *testing.T, i int) []string { path := keepServers[i] + "/index" client := http.Client{} req, err := http.NewRequest("GET", path, nil) - req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil)) + req.Header.Add("Authorization", "OAuth2 " + AdminToken) req.Header.Add("Content-Type", "application/octet-stream") resp, err := client.Do(req) defer resp.Body.Close() @@ -301,7 +303,7 @@ func backdateBlocks(t *testing.T, oldUnusedBlockLocators []string) { func getStatus(t *testing.T, path string) interface{} { client := http.Client{} req, err := http.NewRequest("GET", path, nil) - req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil)) + req.Header.Add("Authorization", "OAuth2 " + AdminToken) req.Header.Add("Content-Type", "application/octet-stream") resp, err := client.Do(req) defer resp.Body.Close() @@ -531,22 +533,10 @@ func TestRunDatamanagerAsNonAdminUser(t *testing.T) { defer TearDownDataManagerTest(t) SetupDataManagerTest(t) - arv.ApiToken = ACTIVE_USER_TOKEN + arv.ApiToken = ActiveUserToken err := singlerun(arv) if err == nil { t.Fatalf("Expected error during singlerun as non-admin user") } } - -func TestRunDatamanagerWithNonAdminDataManagerToken(t *testing.T) { - defer TearDownDataManagerTest(t) - SetupDataManagerTest(t) - - dataManagerToken = ACTIVE_USER_TOKEN - - err := singlerun(arv) - if err == nil { - t.Fatalf("Expected error during singlerun with non-admin user token as datamanager token") - } -} diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go index 0e3cc1d44e..ce316f182b 100644 --- a/services/datamanager/keep/keep.go +++ b/services/datamanager/keep/keep.go @@ -6,7 +6,6 @@ import ( "bufio" "encoding/json" "errors" - "flag" "fmt" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/blockdigest" @@ -19,7 +18,6 @@ import ( "net/http" "strconv" "strings" - "sync" "time" ) @@ -71,21 +69,6 @@ type KeepServiceList struct { KeepServers []ServerAddress `json:"items"` } -var ( - // Don't access the token directly, use getDataManagerToken() to - // make sure it's been read. - dataManagerToken string - dataManagerTokenFile string - dataManagerTokenFileReadOnce sync.Once -) - -func init() { - flag.StringVar(&dataManagerTokenFile, - "data-manager-token-file", - "", - "File with the API token we should use to contact keep servers.") -} - // TODO(misha): Change this to include the UUID as well. func (s ServerAddress) String() string { return s.URL() @@ -99,28 +82,6 @@ func (s ServerAddress) URL() string { } } -func GetDataManagerToken(arvLogger *logger.Logger) string { - readDataManagerToken := func() { - if dataManagerTokenFile == "" { - flag.Usage() - loggerutil.FatalWithMessage(arvLogger, - "Data Manager Token needed, but data manager token file not specified.") - } else { - rawRead, err := ioutil.ReadFile(dataManagerTokenFile) - if err != nil { - loggerutil.FatalWithMessage(arvLogger, - fmt.Sprintf("Unexpected error reading token file %s: %v", - dataManagerTokenFile, - err)) - } - dataManagerToken = strings.TrimSpace(string(rawRead)) - } - } - - dataManagerTokenFileReadOnce.Do(readDataManagerToken) - return dataManagerToken -} - func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServers) { results = GetKeepServers(params) log.Printf("Returned %d keep disks", len(results.ServerToContents)) @@ -177,9 +138,6 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) { log.Printf("Got Server Addresses: %v", results) - // This is safe for concurrent use - client := http.Client{} - // Send off all the index requests concurrently responseChan := make(chan ServerResponse) for _, keepServer := range sdkResponse.KeepServers { @@ -192,7 +150,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) { go func(keepServer ServerAddress) { responseChan <- GetServerContents(params.Logger, keepServer, - client) + params.Client) }(keepServer) } @@ -220,12 +178,12 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) { func GetServerContents(arvLogger *logger.Logger, keepServer ServerAddress, - client http.Client) (response ServerResponse) { + arv arvadosclient.ArvadosClient) (response ServerResponse) { - GetServerStatus(arvLogger, keepServer, client) + GetServerStatus(arvLogger, keepServer, arv) - req := CreateIndexRequest(arvLogger, keepServer) - resp, err := client.Do(req) + req := CreateIndexRequest(arvLogger, keepServer, arv) + resp, err := arv.Client.Do(req) if err != nil { loggerutil.FatalWithMessage(arvLogger, fmt.Sprintf("Error fetching %s: %v. Response was %+v", @@ -239,7 +197,7 @@ func GetServerContents(arvLogger *logger.Logger, func GetServerStatus(arvLogger *logger.Logger, keepServer ServerAddress, - client http.Client) { + arv arvadosclient.ArvadosClient) { url := fmt.Sprintf("http://%s:%d/status.json", keepServer.Host, keepServer.Port) @@ -257,7 +215,7 @@ func GetServerStatus(arvLogger *logger.Logger, }) } - resp, err := client.Get(url) + resp, err := arv.Client.Get(url) if err != nil { loggerutil.FatalWithMessage(arvLogger, fmt.Sprintf("Error getting keep status from %s: %v", url, err)) @@ -289,7 +247,8 @@ func GetServerStatus(arvLogger *logger.Logger, } func CreateIndexRequest(arvLogger *logger.Logger, - keepServer ServerAddress) (req *http.Request) { + keepServer ServerAddress, + arv arvadosclient.ArvadosClient) (req *http.Request) { url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port) log.Println("About to fetch keep server contents from " + url) @@ -308,8 +267,7 @@ func CreateIndexRequest(arvLogger *logger.Logger, fmt.Sprintf("Error building http request for %s: %v", url, err)) } - req.Header.Add("Authorization", - fmt.Sprintf("OAuth2 %s", GetDataManagerToken(arvLogger))) + req.Header.Add("Authorization", "OAuth2 " + arv.ApiToken) return } @@ -462,7 +420,7 @@ type TrashRequest struct { type TrashList []TrashRequest -func SendTrashLists(dataManagerToken string, kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) { +func SendTrashLists(kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) { count := 0 barrier := make(chan error) @@ -487,8 +445,7 @@ func SendTrashLists(dataManagerToken string, kc *keepclient.KeepClient, spl map[ return } - // Add api token header - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", dataManagerToken)) + req.Header.Add("Authorization", "OAuth2 " + kc.Arvados.ApiToken) // Make the request var resp *http.Response