"time"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/controller/dblock"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/lib/dispatchcloud"
"git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
Registry *prometheus.Registry
logger logrus.FieldLogger
+ dbConnector ctrlctx.DBConnector
lsfcli lsfcli
lsfqueue lsfqueue
arvDispatcher *dispatch.Dispatcher
func (disp *dispatcher) Start() {
disp.initOnce.Do(func() {
disp.init()
+ dblock.Dispatch.Lock(context.Background(), disp.dbConnector.GetDB)
go func() {
+ defer dblock.Dispatch.Unlock()
disp.checkLsfQueueForOrphans()
err := disp.arvDispatcher.Run(disp.Context)
if err != nil {
lsfcli: &disp.lsfcli,
}
disp.ArvClient.AuthToken = disp.AuthToken
+ disp.dbConnector = ctrlctx.DBConnector{PostgreSQL: disp.Cluster.PostgreSQL}
disp.stop = make(chan struct{}, 1)
disp.stopped = make(chan struct{})
container.RuntimeConstraints.KeepCacheRAM+
int64(disp.Cluster.Containers.ReserveExtraRAM)) / 1048576))
+ maxruntime := time.Duration(container.SchedulingParameters.MaxRunTime) * time.Second
+ if maxruntime == 0 {
+ maxruntime = disp.Cluster.Containers.LSF.MaxRunTimeDefault.Duration()
+ }
+ if maxruntime > 0 {
+ maxruntime += disp.Cluster.Containers.LSF.MaxRunTimeOverhead.Duration()
+ }
+ maxrunminutes := int64(math.Ceil(float64(maxruntime.Seconds()) / 60))
+
repl := map[string]string{
"%%": "%",
"%C": fmt.Sprintf("%d", vcpus),
"%T": fmt.Sprintf("%d", tmp),
"%U": container.UUID,
"%G": fmt.Sprintf("%d", container.RuntimeConstraints.CUDA.DeviceCount),
+ "%W": fmt.Sprintf("%d", maxrunminutes),
}
re := regexp.MustCompile(`%.`)
if container.RuntimeConstraints.CUDA.DeviceCount > 0 {
argumentTemplate = append(argumentTemplate, disp.Cluster.Containers.LSF.BsubCUDAArguments...)
}
- for _, a := range argumentTemplate {
+ for idx, a := range argumentTemplate {
+ if idx > 0 && (argumentTemplate[idx-1] == "-W" || argumentTemplate[idx-1] == "-We") && a == "%W" && maxrunminutes == 0 {
+ // LSF docs don't specify an argument to "-W"
+ // or "-We" that indicates "unknown", so
+ // instead we drop the "-W %W" part of the
+ // command line entirely when max runtime is
+ // unknown.
+ args = args[:len(args)-1]
+ continue
+ }
args = append(args, re.ReplaceAllStringFunc(a, func(s string) string {
subst := repl[s]
if len(subst) == 0 {