6260: Just use the token loaded by the SDK (ARVADOS_API_TOKEN),

instead of requiring a token separately on the command line.

diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 9cfd2ee5d71891fd6dfb1ff99f496d314abcdf7c..3fb135cb3aaa54c3e2ee2ff1a22519c159225cc2 100644 (file)
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -68,8 +68,6 @@ func makeArvadosClient() arvadosclient.ArvadosClient {
        return arv
 }
 
-var dataManagerToken string
-
 func singlerun(arv arvadosclient.ArvadosClient) error {
        var err error
        if is_admin, err := util.UserIsAdmin(arv); err != nil {
@@ -93,21 +91,6 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
                arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
        }
 
-       // Verify that datamanager token belongs to an admin user
-       if dataManagerToken == "" {
-               dataManagerToken = keep.GetDataManagerToken(arvLogger)
-       }
-       origArvToken := arv.ApiToken
-       arv.ApiToken = dataManagerToken
-       if is_admin, err := util.UserIsAdmin(arv); err != nil {
-               log.Printf("Error querying arvados user for data manager token %s", err.Error())
-               return err
-       } else if !is_admin {
-               log.Printf("Datamanager token does not belong to an admin user.")
-               return errors.New("Datamanager token does not belong to an admin user.")
-       }
-       arv.ApiToken = origArvToken
-
        var (
                dataFetcher     summary.DataFetcher
                readCollections collection.ReadCollections
@@ -178,7 +161,7 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
        if trashErr != nil {
                return err
        } else {
-               keep.SendTrashLists(dataManagerToken, kc, trashLists)
+               keep.SendTrashLists(kc, trashLists)
        }
 
        return nil

diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 2ab1a481598042708d71bffba455140ab85e66e8..18b302a5ba68580568016a29720bbbc6185cf747 100644 (file)
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -6,7 +6,6 @@ import (
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/arvadostest"
        "git.curoverse.com/arvados.git/sdk/go/keepclient"
-       "git.curoverse.com/arvados.git/services/datamanager/keep"
        "io/ioutil"
        "net/http"
        "os"
@@ -17,7 +16,10 @@ import (
        "time"
 )
 
-const ACTIVE_USER_TOKEN = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+const (
+       ActiveUserToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+       AdminToken = "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h"
+)
 
 var arv arvadosclient.ArvadosClient
 var keepClient *keepclient.KeepClient
@@ -172,7 +174,7 @@ func getBlockIndexesForServer(t *testing.T, i int) []string {
        path := keepServers[i] + "/index"
        client := http.Client{}
        req, err := http.NewRequest("GET", path, nil)
-       req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
+       req.Header.Add("Authorization", "OAuth2 " + AdminToken)
        req.Header.Add("Content-Type", "application/octet-stream")
        resp, err := client.Do(req)
        defer resp.Body.Close()
@@ -301,7 +303,7 @@ func backdateBlocks(t *testing.T, oldUnusedBlockLocators []string) {
 func getStatus(t *testing.T, path string) interface{} {
        client := http.Client{}
        req, err := http.NewRequest("GET", path, nil)
-       req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
+       req.Header.Add("Authorization", "OAuth2 " + AdminToken)
        req.Header.Add("Content-Type", "application/octet-stream")
        resp, err := client.Do(req)
        defer resp.Body.Close()
@@ -531,22 +533,10 @@ func TestRunDatamanagerAsNonAdminUser(t *testing.T) {
        defer TearDownDataManagerTest(t)
        SetupDataManagerTest(t)
 
-       arv.ApiToken = ACTIVE_USER_TOKEN
+       arv.ApiToken = ActiveUserToken
 
        err := singlerun(arv)
        if err == nil {
                t.Fatalf("Expected error during singlerun as non-admin user")
        }
 }
-
-func TestRunDatamanagerWithNonAdminDataManagerToken(t *testing.T) {
-       defer TearDownDataManagerTest(t)
-       SetupDataManagerTest(t)
-
-       dataManagerToken = ACTIVE_USER_TOKEN
-
-       err := singlerun(arv)
-       if err == nil {
-               t.Fatalf("Expected error during singlerun with non-admin user token as datamanager token")
-       }
-}

diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 0e3cc1d44e79ef7b533a8cccef77a7c5cf6c6605..ce316f182b881755f097ed74250f10ffd41919ea 100644 (file)
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -6,7 +6,6 @@ import (
        "bufio"
        "encoding/json"
        "errors"
-       "flag"
        "fmt"
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/blockdigest"
@@ -19,7 +18,6 @@ import (
        "net/http"
        "strconv"
        "strings"
-       "sync"
        "time"
 )
 
@@ -71,21 +69,6 @@ type KeepServiceList struct {
        KeepServers    []ServerAddress `json:"items"`
 }
 
-var (
-       // Don't access the token directly, use getDataManagerToken() to
-       // make sure it's been read.
-       dataManagerToken             string
-       dataManagerTokenFile         string
-       dataManagerTokenFileReadOnce sync.Once
-)
-
-func init() {
-       flag.StringVar(&dataManagerTokenFile,
-               "data-manager-token-file",
-               "",
-               "File with the API token we should use to contact keep servers.")
-}
-
 // TODO(misha): Change this to include the UUID as well.
 func (s ServerAddress) String() string {
        return s.URL()
@@ -99,28 +82,6 @@ func (s ServerAddress) URL() string {
        }
 }
 
-func GetDataManagerToken(arvLogger *logger.Logger) string {
-       readDataManagerToken := func() {
-               if dataManagerTokenFile == "" {
-                       flag.Usage()
-                       loggerutil.FatalWithMessage(arvLogger,
-                               "Data Manager Token needed, but data manager token file not specified.")
-               } else {
-                       rawRead, err := ioutil.ReadFile(dataManagerTokenFile)
-                       if err != nil {
-                               loggerutil.FatalWithMessage(arvLogger,
-                                       fmt.Sprintf("Unexpected error reading token file %s: %v",
-                                               dataManagerTokenFile,
-                                               err))
-                       }
-                       dataManagerToken = strings.TrimSpace(string(rawRead))
-               }
-       }
-
-       dataManagerTokenFileReadOnce.Do(readDataManagerToken)
-       return dataManagerToken
-}
-
 func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServers) {
        results = GetKeepServers(params)
        log.Printf("Returned %d keep disks", len(results.ServerToContents))
@@ -177,9 +138,6 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
 
        log.Printf("Got Server Addresses: %v", results)
 
-       // This is safe for concurrent use
-       client := http.Client{}
-
        // Send off all the index requests concurrently
        responseChan := make(chan ServerResponse)
        for _, keepServer := range sdkResponse.KeepServers {
@@ -192,7 +150,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
                go func(keepServer ServerAddress) {
                        responseChan <- GetServerContents(params.Logger,
                                keepServer,
-                               client)
+                               params.Client)
                }(keepServer)
        }
 
@@ -220,12 +178,12 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
 
 func GetServerContents(arvLogger *logger.Logger,
        keepServer ServerAddress,
-       client http.Client) (response ServerResponse) {
+       arv arvadosclient.ArvadosClient) (response ServerResponse) {
 
-       GetServerStatus(arvLogger, keepServer, client)
+       GetServerStatus(arvLogger, keepServer, arv)
 
-       req := CreateIndexRequest(arvLogger, keepServer)
-       resp, err := client.Do(req)
+       req := CreateIndexRequest(arvLogger, keepServer, arv)
+       resp, err := arv.Client.Do(req)
        if err != nil {
                loggerutil.FatalWithMessage(arvLogger,
                        fmt.Sprintf("Error fetching %s: %v. Response was %+v",
@@ -239,7 +197,7 @@ func GetServerContents(arvLogger *logger.Logger,
 
 func GetServerStatus(arvLogger *logger.Logger,
        keepServer ServerAddress,
-       client http.Client) {
+       arv arvadosclient.ArvadosClient) {
        url := fmt.Sprintf("http://%s:%d/status.json",
                keepServer.Host,
                keepServer.Port)
@@ -257,7 +215,7 @@ func GetServerStatus(arvLogger *logger.Logger,
                })
        }
 
-       resp, err := client.Get(url)
+       resp, err := arv.Client.Get(url)
        if err != nil {
                loggerutil.FatalWithMessage(arvLogger,
                        fmt.Sprintf("Error getting keep status from %s: %v", url, err))
@@ -289,7 +247,8 @@ func GetServerStatus(arvLogger *logger.Logger,
 }
 
 func CreateIndexRequest(arvLogger *logger.Logger,
-       keepServer ServerAddress) (req *http.Request) {
+       keepServer ServerAddress,
+       arv arvadosclient.ArvadosClient) (req *http.Request) {
        url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
        log.Println("About to fetch keep server contents from " + url)
 
@@ -308,8 +267,7 @@ func CreateIndexRequest(arvLogger *logger.Logger,
                        fmt.Sprintf("Error building http request for %s: %v", url, err))
        }
 
-       req.Header.Add("Authorization",
-               fmt.Sprintf("OAuth2 %s", GetDataManagerToken(arvLogger)))
+       req.Header.Add("Authorization", "OAuth2 " + arv.ApiToken)
        return
 }
 
@@ -462,7 +420,7 @@ type TrashRequest struct {
 
 type TrashList []TrashRequest
 
-func SendTrashLists(dataManagerToken string, kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) {
+func SendTrashLists(kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) {
        count := 0
        barrier := make(chan error)
 
@@ -487,8 +445,7 @@ func SendTrashLists(dataManagerToken string, kc *keepclient.KeepClient, spl map[
                                return
                        }
 
-                       // Add api token header
-                       req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", dataManagerToken))
+                       req.Header.Add("Authorization", "OAuth2 " + kc.Arvados.ApiToken)
 
                        // Make the request
                        var resp *http.Response