projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
17816: Add --runtime-engine to crunch-dispatch-local and crunch-dispatch-slurm
[arvados.git]
/
services
/
crunch-dispatch-local
/
crunch-dispatch-local.go
diff --git
a/services/crunch-dispatch-local/crunch-dispatch-local.go
b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 279327ba18811ba8ad6339600cc124460f2fc35c..4a45f10545f9165463d3f80384de0f12610f5d0d 100644
(file)
--- a/
services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/
services/crunch-dispatch-local/crunch-dispatch-local.go
@@
-10,7
+10,6
@@
import (
"context"
"flag"
"fmt"
"context"
"flag"
"fmt"
- "log"
"os"
"os/exec"
"os/signal"
"os"
"os/exec"
"os/signal"
@@
-18,9
+17,11
@@
import (
"syscall"
"time"
"syscall"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/dispatch"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/dispatch"
+ "github.com/sirupsen/logrus"
)
var version = "dev"
)
var version = "dev"
@@
-28,7
+29,7
@@
var version = "dev"
func main() {
err := doMain()
if err != nil {
func main() {
err := doMain()
if err != nil {
- log.Fatalf("%q", err)
+ log
rus
.Fatalf("%q", err)
}
}
}
}
@@
-40,6
+41,14
@@
var (
)
func doMain() error {
)
func doMain() error {
+ logger := logrus.StandardLogger()
+ if os.Getenv("DEBUG") != "" {
+ logger.SetLevel(logrus.DebugLevel)
+ }
+ logger.Formatter = &logrus.JSONFormatter{
+ TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
+ }
+
flags := flag.NewFlagSet("crunch-dispatch-local", flag.ExitOnError)
pollInterval := flags.Int(
flags := flag.NewFlagSet("crunch-dispatch-local", flag.ExitOnError)
pollInterval := flags.Int(
@@
-66,24
+75,33
@@
func doMain() error {
return nil
}
return nil
}
- log.Printf("crunch-dispatch-local %s started", version)
+ loader := config.NewLoader(nil, logger)
+ cfg, err := loader.Load()
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return fmt.Errorf("config error: %s", err)
+ }
+
+ logger.Printf("crunch-dispatch-local %s started", version)
runningCmds = make(map[string]*exec.Cmd)
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
runningCmds = make(map[string]*exec.Cmd)
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log
.Printf("E
rror making Arvados client: %v", err)
+ log
ger.Errorf("e
rror making Arvados client: %v", err)
return err
}
arv.Retries = 25
return err
}
arv.Retries = 25
+ ctx, cancel := context.WithCancel(context.Background())
+
dispatcher := dispatch.Dispatcher{
dispatcher := dispatch.Dispatcher{
+ Logger: logger,
Arv: arv,
Arv: arv,
- RunContainer: run,
+ RunContainer:
(&LocalRun{startFunc, make(chan bool, 8), ctx, cluster}).
run,
PollPeriod: time.Duration(*pollInterval) * time.Second,
}
PollPeriod: time.Duration(*pollInterval) * time.Second,
}
- ctx, cancel := context.WithCancel(context.Background())
err = dispatcher.Run(ctx)
if err != nil {
return err
err = dispatcher.Run(ctx)
if err != nil {
return err
@@
-92,7
+110,7
@@
func doMain() error {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
sig := <-c
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
sig := <-c
- log.Printf("Received %s, shutting down", sig)
+ log
ger
.Printf("Received %s, shutting down", sig)
signal.Stop(c)
cancel()
signal.Stop(c)
cancel()
@@
-114,7
+132,12
@@
func startFunc(container arvados.Container, cmd *exec.Cmd) error {
return cmd.Start()
}
return cmd.Start()
}
-var startCmd = startFunc
+type LocalRun struct {
+ startCmd func(container arvados.Container, cmd *exec.Cmd) error
+ concurrencyLimit chan bool
+ ctx context.Context
+ cluster *arvados.Cluster
+}
// Run a container.
//
// Run a container.
//
@@
-124,29
+147,51
@@
var startCmd = startFunc
//
// If the container is in any other state, or is not Complete/Cancelled after
// crunch-run terminates, mark the container as Cancelled.
//
// If the container is in any other state, or is not Complete/Cancelled after
// crunch-run terminates, mark the container as Cancelled.
-func run(dispatcher *dispatch.Dispatcher,
+func
(lr *LocalRun)
run(dispatcher *dispatch.Dispatcher,
container arvados.Container,
status <-chan arvados.Container) {
uuid := container.UUID
if container.State == dispatch.Locked {
container arvados.Container,
status <-chan arvados.Container) {
uuid := container.UUID
if container.State == dispatch.Locked {
+
+ select {
+ case lr.concurrencyLimit <- true:
+ break
+ case <-lr.ctx.Done():
+ return
+ }
+
+ defer func() { <-lr.concurrencyLimit }()
+
+ select {
+ case c := <-status:
+ // Check for state updates after possibly
+ // waiting to be ready-to-run
+ if c.Priority == 0 {
+ goto Finish
+ }
+ default:
+ break
+ }
+
waitGroup.Add(1)
waitGroup.Add(1)
+ defer waitGroup.Done()
- cmd := exec.Command(*crunchRunCommand, uuid)
+ cmd := exec.Command(*crunchRunCommand,
"--runtime-engine="+lr.cluster.Containers.RuntimeEngine,
uuid)
cmd.Stdin = nil
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
cmd.Stdin = nil
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
-
log.Printf("S
tarting container %v", uuid)
+
dispatcher.Logger.Printf("s
tarting container %v", uuid)
// Add this crunch job to the list of runningCmds only if we
// succeed in starting crunch-run.
runningCmdsMutex.Lock()
// Add this crunch job to the list of runningCmds only if we
// succeed in starting crunch-run.
runningCmdsMutex.Lock()
- if err := startCmd(container, cmd); err != nil {
+ if err :=
lr.
startCmd(container, cmd); err != nil {
runningCmdsMutex.Unlock()
runningCmdsMutex.Unlock()
-
log.Printf("Error starting %v for %v: %q
", *crunchRunCommand, uuid, err)
+
dispatcher.Logger.Warnf("error starting %q for %s: %s
", *crunchRunCommand, uuid, err)
dispatcher.UpdateState(uuid, dispatch.Cancelled)
} else {
runningCmds[uuid] = cmd
dispatcher.UpdateState(uuid, dispatch.Cancelled)
} else {
runningCmds[uuid] = cmd
@@
-157,9
+202,9
@@
func run(dispatcher *dispatch.Dispatcher,
go func() {
if _, err := cmd.Process.Wait(); err != nil {
go func() {
if _, err := cmd.Process.Wait(); err != nil {
-
log.Printf("E
rror while waiting for crunch job to finish for %v: %q", uuid, err)
+
dispatcher.Logger.Warnf("e
rror while waiting for crunch job to finish for %v: %q", uuid, err)
}
}
-
log.Print
f("sending done")
+
dispatcher.Logger.Debug
f("sending done")
done <- struct{}{}
}()
done <- struct{}{}
}()
@@
-171,31
+216,32
@@
func run(dispatcher *dispatch.Dispatcher,
case c := <-status:
// Interrupt the child process if priority changes to 0
if (c.State == dispatch.Locked || c.State == dispatch.Running) && c.Priority == 0 {
case c := <-status:
// Interrupt the child process if priority changes to 0
if (c.State == dispatch.Locked || c.State == dispatch.Running) && c.Priority == 0 {
-
log.Printf("S
ending SIGINT to pid %d to cancel container %v", cmd.Process.Pid, uuid)
+
dispatcher.Logger.Printf("s
ending SIGINT to pid %d to cancel container %v", cmd.Process.Pid, uuid)
cmd.Process.Signal(os.Interrupt)
}
}
}
close(done)
cmd.Process.Signal(os.Interrupt)
}
}
}
close(done)
-
log.Printf("F
inished container run for %v", uuid)
+
dispatcher.Logger.Printf("f
inished container run for %v", uuid)
// Remove the crunch job from runningCmds
runningCmdsMutex.Lock()
delete(runningCmds, uuid)
runningCmdsMutex.Unlock()
}
// Remove the crunch job from runningCmds
runningCmdsMutex.Lock()
delete(runningCmds, uuid)
runningCmdsMutex.Unlock()
}
- waitGroup.Done()
}
}
+Finish:
+
// If the container is not finalized, then change it to "Cancelled".
err := dispatcher.Arv.Get("containers", uuid, nil, &container)
if err != nil {
// If the container is not finalized, then change it to "Cancelled".
err := dispatcher.Arv.Get("containers", uuid, nil, &container)
if err != nil {
-
log.Printf("E
rror getting final container state: %v", err)
+
dispatcher.Logger.Warnf("e
rror getting final container state: %v", err)
}
if container.State == dispatch.Locked || container.State == dispatch.Running {
}
if container.State == dispatch.Locked || container.State == dispatch.Running {
-
log.Printf("After %s process termination, container state for %v is %q. U
pdating it to %q",
- *crunchRunCommand,
container.State, uuid
, dispatch.Cancelled)
+
dispatcher.Logger.Warnf("after %q process termination, container state for %v is %q; u
pdating it to %q",
+ *crunchRunCommand,
uuid, container.State
, dispatch.Cancelled)
dispatcher.UpdateState(uuid, dispatch.Cancelled)
}
dispatcher.UpdateState(uuid, dispatch.Cancelled)
}
@@
-203,5
+249,5
@@
func run(dispatcher *dispatch.Dispatcher,
for range status {
}
for range status {
}
-
log.Printf("F
inalized container %v", uuid)
+
dispatcher.Logger.Printf("f
inalized container %v", uuid)
}
}