projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch 'master' into 5694-running-job-log
[arvados.git]
/
services
/
datamanager
/
datamanager.go
diff --git
a/services/datamanager/datamanager.go
b/services/datamanager/datamanager.go
index 4e3b4e95671c7a63f54be270e88df1db5cbf7b7f..604a6db2f4a16ac97aea4579111f0d909c6c5d9e 100644
(file)
--- a/
services/datamanager/datamanager.go
+++ b/
services/datamanager/datamanager.go
@@
-22,6
+22,7
@@
var (
logEventTypePrefix string
logFrequencySeconds int
minutesBetweenRuns int
logEventTypePrefix string
logFrequencySeconds int
minutesBetweenRuns int
+ dryRun bool
)
func init() {
)
func init() {
@@
-36,21
+37,34
@@
func init() {
flag.IntVar(&minutesBetweenRuns,
"minutes-between-runs",
0,
flag.IntVar(&minutesBetweenRuns,
"minutes-between-runs",
0,
- "How many minutes we wait betwen data manager runs. 0 means run once and exit.")
+ "How many minutes we wait between data manager runs. 0 means run once and exit.")
+ flag.BoolVar(&dryRun,
+ "dry-run",
+ false,
+ "Perform a dry run. Log how many blocks would be deleted/moved, but do not issue any changes to keepstore.")
}
func main() {
flag.Parse()
}
func main() {
flag.Parse()
+
if minutesBetweenRuns == 0 {
if minutesBetweenRuns == 0 {
- err := singlerun(makeArvadosClient())
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ loggerutil.FatalWithMessage(arvLogger, fmt.Sprintf("Error making arvados client: %v", err))
+ }
+ err = singlerun(arv)
if err != nil {
if err != nil {
- log
.Fatalf("singlerun: %v", err
)
+ log
gerutil.FatalWithMessage(arvLogger, fmt.Sprintf("singlerun: %v", err)
)
}
} else {
waitTime := time.Minute * time.Duration(minutesBetweenRuns)
for {
log.Println("Beginning Run")
}
} else {
waitTime := time.Minute * time.Duration(minutesBetweenRuns)
for {
log.Println("Beginning Run")
- err := singlerun(makeArvadosClient())
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ loggerutil.FatalWithMessage(arvLogger, fmt.Sprintf("Error making arvados client: %v", err))
+ }
+ err = singlerun(arv)
if err != nil {
log.Printf("singlerun: %v", err)
}
if err != nil {
log.Printf("singlerun: %v", err)
}
@@
-60,13
+74,7
@@
func main() {
}
}
}
}
-func makeArvadosClient() arvadosclient.ArvadosClient {
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Fatalf("Error setting up arvados client: %s", err)
- }
- return arv
-}
+var arvLogger *logger.Logger
func singlerun(arv arvadosclient.ArvadosClient) error {
var err error
func singlerun(arv arvadosclient.ArvadosClient) error {
var err error
@@
-76,9
+84,8
@@
func singlerun(arv arvadosclient.ArvadosClient) error {
return errors.New("Current user is not an admin. Datamanager requires a privileged token.")
}
return errors.New("Current user is not an admin. Datamanager requires a privileged token.")
}
- var arvLogger *logger.Logger
if logEventTypePrefix != "" {
if logEventTypePrefix != "" {
- arvLogger = logger.NewLogger(logger.LoggerParams{
+ arvLogger
, err
= logger.NewLogger(logger.LoggerParams{
Client: arv,
EventTypePrefix: logEventTypePrefix,
WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
Client: arv,
EventTypePrefix: logEventTypePrefix,
WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
@@
-101,9
+108,15
@@
func singlerun(arv arvadosclient.ArvadosClient) error {
dataFetcher = BuildDataFetcher(arv)
}
dataFetcher = BuildDataFetcher(arv)
}
- dataFetcher(arvLogger, &readCollections, &keepServerInfo)
+ err = dataFetcher(arvLogger, &readCollections, &keepServerInfo)
+ if err != nil {
+ return err
+ }
- summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
+ err = summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
+ if err != nil {
+ return err
+ }
buckets := summary.BucketReplication(readCollections, keepServerInfo)
bucketCounts := buckets.Counts()
buckets := summary.BucketReplication(readCollections, keepServerInfo)
bucketCounts := buckets.Counts()
@@
-126,8
+139,7
@@
func singlerun(arv arvadosclient.ArvadosClient) error {
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
- loggerutil.FatalWithMessage(arvLogger,
- fmt.Sprintf("Error setting up keep client %s", err.Error()))
+ return fmt.Errorf("Error setting up keep client %v", err.Error())
}
// Log that we're finished. We force the recording, since go will
}
// Log that we're finished. We force the recording, since go will
@@
-154,38
+166,50
@@
func singlerun(arv arvadosclient.ArvadosClient) error {
&keepServerInfo,
replicationSummary.KeepBlocksNotInCollections)
&keepServerInfo,
replicationSummary.KeepBlocksNotInCollections)
- summary.WritePullLists(arvLogger, pullLists)
+ err = summary.WritePullLists(arvLogger, pullLists, dryRun)
+ if err != nil {
+ return err
+ }
if trashErr != nil {
return err
if trashErr != nil {
return err
- } else {
- keep.SendTrashLists(kc, trashLists)
}
}
+ keep.SendTrashLists(arvLogger, kc, trashLists, dryRun)
return nil
}
return nil
}
-//
R
eturns a data fetcher that fetches data from remote servers.
+//
BuildDataFetcher r
eturns a data fetcher that fetches data from remote servers.
func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
- return func(arvLogger *logger.Logger,
+ return func(
+ arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
readCollections *collection.ReadCollections,
- keepServerInfo *keep.ReadServers) {
- collectionChannel := make(chan collection.ReadCollections)
-
+ keepServerInfo *keep.ReadServers,
+ ) error {
+ collDone := make(chan struct{})
+ var collErr error
go func() {
go func() {
-
collectionChannel <-
collection.GetCollectionsAndSummarize(
+
*readCollections, collErr =
collection.GetCollectionsAndSummarize(
collection.GetCollectionsParams{
Client: arv,
Logger: arvLogger,
BatchSize: 50})
collection.GetCollectionsParams{
Client: arv,
Logger: arvLogger,
BatchSize: 50})
+ collDone <- struct{}{}
}()
}()
- *keepServerInfo = keep.GetKeepServersAndSummarize(
+ var keepErr error
+ *keepServerInfo, keepErr = keep.GetKeepServersAndSummarize(
keep.GetKeepServersParams{
Client: arv,
Logger: arvLogger,
Limit: 1000})
keep.GetKeepServersParams{
Client: arv,
Logger: arvLogger,
Limit: 1000})
- *readCollections = <-collectionChannel
+ <- collDone
+
+ // Return a nil error only if both parts succeeded.
+ if collErr != nil {
+ return collErr
+ }
+ return keepErr
}
}
}
}