projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
9701: Use a collection.OrderedDict instead of a simple dict to hold bufferblocks...
[arvados.git]
/
services
/
datamanager
/
datamanager.go
diff --git
a/services/datamanager/datamanager.go
b/services/datamanager/datamanager.go
index b26e66ab3703b42d81682019c9367beeb0598178..5250d175ffa9995779c47d2fbaefedc992f09096 100644
(file)
--- a/
services/datamanager/datamanager.go
+++ b/
services/datamanager/datamanager.go
@@
-22,6
+22,7
@@
var (
logEventTypePrefix string
logFrequencySeconds int
minutesBetweenRuns int
logEventTypePrefix string
logFrequencySeconds int
minutesBetweenRuns int
+ collectionBatchSize int
dryRun bool
)
dryRun bool
)
@@
-38,6
+39,10
@@
func init() {
"minutes-between-runs",
0,
"How many minutes we wait between data manager runs. 0 means run once and exit.")
"minutes-between-runs",
0,
"How many minutes we wait between data manager runs. 0 means run once and exit.")
+ flag.IntVar(&collectionBatchSize,
+ "collection-batch-size",
+ 1000,
+ "How many collections to request in each batch.")
flag.BoolVar(&dryRun,
"dry-run",
false,
flag.BoolVar(&dryRun,
"dry-run",
false,
@@
-76,7
+81,7
@@
func main() {
var arvLogger *logger.Logger
var arvLogger *logger.Logger
-func singlerun(arv arvadosclient.ArvadosClient) error {
+func singlerun(arv
*
arvadosclient.ArvadosClient) error {
var err error
if isAdmin, err := util.UserIsAdmin(arv); err != nil {
return errors.New("Error verifying admin token: " + err.Error())
var err error
if isAdmin, err := util.UserIsAdmin(arv); err != nil {
return errors.New("Error verifying admin token: " + err.Error())
@@
-137,7
+142,7
@@
func singlerun(arv arvadosclient.ArvadosClient) error {
rlbss.Count)
}
rlbss.Count)
}
- kc, err := keepclient.MakeKeepClient(
&
arv)
+ kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
return fmt.Errorf("Error setting up keep client %v", err.Error())
}
if err != nil {
return fmt.Errorf("Error setting up keep client %v", err.Error())
}
@@
-180,7
+185,7
@@
func singlerun(arv arvadosclient.ArvadosClient) error {
}
// BuildDataFetcher returns a data fetcher that fetches data from remote servers.
}
// BuildDataFetcher returns a data fetcher that fetches data from remote servers.
-func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
+func BuildDataFetcher(arv
*
arvadosclient.ArvadosClient) summary.DataFetcher {
return func(
arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
return func(
arvLogger *logger.Logger,
readCollections *collection.ReadCollections,
@@
-193,7
+198,7
@@
func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
collection.GetCollectionsParams{
Client: arv,
Logger: arvLogger,
collection.GetCollectionsParams{
Client: arv,
Logger: arvLogger,
- BatchSize:
1000
})
+ BatchSize:
collectionBatchSize
})
collDone <- struct{}{}
}()
collDone <- struct{}{}
}()
@@
-204,7
+209,7
@@
func BuildDataFetcher(arv arvadosclient.ArvadosClient) summary.DataFetcher {
Logger: arvLogger,
Limit: 1000})
Logger: arvLogger,
Limit: 1000})
- <-
collDone
+ <-collDone
// Return a nil error only if both parts succeeded.
if collErr != nil {
// Return a nil error only if both parts succeeded.
if collErr != nil {