projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch '13973-child-priority' refs #13973
[arvados.git]
/
services
/
crunch-dispatch-local
/
crunch-dispatch-local.go
diff --git
a/services/crunch-dispatch-local/crunch-dispatch-local.go
b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 22f7d8b6469bfdd65497a18eff738850d94bc1c6..fc10393626be103c17b01b5b1bfde615ed470bc9 100644
(file)
--- a/
services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/
services/crunch-dispatch-local/crunch-dispatch-local.go
@@
-1,3
+1,7
@@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package main
// Dispatcher service for Crunch that runs containers locally.
package main
// Dispatcher service for Crunch that runs containers locally.
@@
-5,7
+9,7
@@
package main
import (
"context"
"flag"
import (
"context"
"flag"
- "
log
"
+ "
fmt
"
"os"
"os/exec"
"os/signal"
"os"
"os/exec"
"os/signal"
@@
-16,12
+20,15
@@
import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/dispatch"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/dispatch"
+ "github.com/Sirupsen/logrus"
)
)
+var version = "dev"
+
func main() {
err := doMain()
if err != nil {
func main() {
err := doMain()
if err != nil {
- log.Fatalf("%q", err)
+ log
rus
.Fatalf("%q", err)
}
}
}
}
@@
-33,6
+40,14
@@
var (
)
func doMain() error {
)
func doMain() error {
+ logger := logrus.StandardLogger()
+ if os.Getenv("DEBUG") != "" {
+ logger.SetLevel(logrus.DebugLevel)
+ }
+ logger.Formatter = &logrus.JSONFormatter{
+ TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
+ }
+
flags := flag.NewFlagSet("crunch-dispatch-local", flag.ExitOnError)
pollInterval := flags.Int(
flags := flag.NewFlagSet("crunch-dispatch-local", flag.ExitOnError)
pollInterval := flags.Int(
@@
-45,19
+60,33
@@
func doMain() error {
"/usr/bin/crunch-run",
"Crunch command to run container")
"/usr/bin/crunch-run",
"Crunch command to run container")
+ getVersion := flags.Bool(
+ "version",
+ false,
+ "Print version information and exit.")
+
// Parse args; omit the first arg which is the command name
flags.Parse(os.Args[1:])
// Parse args; omit the first arg which is the command name
flags.Parse(os.Args[1:])
+ // Print version information if requested
+ if *getVersion {
+ fmt.Printf("crunch-dispatch-local %s\n", version)
+ return nil
+ }
+
+ logger.Printf("crunch-dispatch-local %s started", version)
+
runningCmds = make(map[string]*exec.Cmd)
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
runningCmds = make(map[string]*exec.Cmd)
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log
.Printf("E
rror making Arvados client: %v", err)
+ log
ger.Errorf("e
rror making Arvados client: %v", err)
return err
}
arv.Retries = 25
dispatcher := dispatch.Dispatcher{
return err
}
arv.Retries = 25
dispatcher := dispatch.Dispatcher{
+ Logger: logger,
Arv: arv,
RunContainer: run,
PollPeriod: time.Duration(*pollInterval) * time.Second,
Arv: arv,
RunContainer: run,
PollPeriod: time.Duration(*pollInterval) * time.Second,
@@
-72,7
+101,7
@@
func doMain() error {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
sig := <-c
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
sig := <-c
- log.Printf("Received %s, shutting down", sig)
+ log
ger
.Printf("Received %s, shutting down", sig)
signal.Stop(c)
cancel()
signal.Stop(c)
cancel()
@@
-118,7
+147,7
@@
func run(dispatcher *dispatch.Dispatcher,
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
-
log.Printf("S
tarting container %v", uuid)
+
dispatcher.Logger.Printf("s
tarting container %v", uuid)
// Add this crunch job to the list of runningCmds only if we
// succeed in starting crunch-run.
// Add this crunch job to the list of runningCmds only if we
// succeed in starting crunch-run.
@@
-126,7
+155,7
@@
func run(dispatcher *dispatch.Dispatcher,
runningCmdsMutex.Lock()
if err := startCmd(container, cmd); err != nil {
runningCmdsMutex.Unlock()
runningCmdsMutex.Lock()
if err := startCmd(container, cmd); err != nil {
runningCmdsMutex.Unlock()
-
log.Printf("Error starting %v for %v: %q
", *crunchRunCommand, uuid, err)
+
dispatcher.Logger.Warnf("error starting %q for %s: %s
", *crunchRunCommand, uuid, err)
dispatcher.UpdateState(uuid, dispatch.Cancelled)
} else {
runningCmds[uuid] = cmd
dispatcher.UpdateState(uuid, dispatch.Cancelled)
} else {
runningCmds[uuid] = cmd
@@
-137,9
+166,9
@@
func run(dispatcher *dispatch.Dispatcher,
go func() {
if _, err := cmd.Process.Wait(); err != nil {
go func() {
if _, err := cmd.Process.Wait(); err != nil {
-
log.Printf("E
rror while waiting for crunch job to finish for %v: %q", uuid, err)
+
dispatcher.Logger.Warnf("e
rror while waiting for crunch job to finish for %v: %q", uuid, err)
}
}
-
log.Print
f("sending done")
+
dispatcher.Logger.Debug
f("sending done")
done <- struct{}{}
}()
done <- struct{}{}
}()
@@
-151,14
+180,14
@@
func run(dispatcher *dispatch.Dispatcher,
case c := <-status:
// Interrupt the child process if priority changes to 0
if (c.State == dispatch.Locked || c.State == dispatch.Running) && c.Priority == 0 {
case c := <-status:
// Interrupt the child process if priority changes to 0
if (c.State == dispatch.Locked || c.State == dispatch.Running) && c.Priority == 0 {
-
log.Printf("S
ending SIGINT to pid %d to cancel container %v", cmd.Process.Pid, uuid)
+
dispatcher.Logger.Printf("s
ending SIGINT to pid %d to cancel container %v", cmd.Process.Pid, uuid)
cmd.Process.Signal(os.Interrupt)
}
}
}
close(done)
cmd.Process.Signal(os.Interrupt)
}
}
}
close(done)
-
log.Printf("F
inished container run for %v", uuid)
+
dispatcher.Logger.Printf("f
inished container run for %v", uuid)
// Remove the crunch job from runningCmds
runningCmdsMutex.Lock()
// Remove the crunch job from runningCmds
runningCmdsMutex.Lock()
@@
-171,11
+200,11
@@
func run(dispatcher *dispatch.Dispatcher,
// If the container is not finalized, then change it to "Cancelled".
err := dispatcher.Arv.Get("containers", uuid, nil, &container)
if err != nil {
// If the container is not finalized, then change it to "Cancelled".
err := dispatcher.Arv.Get("containers", uuid, nil, &container)
if err != nil {
-
log.Printf("E
rror getting final container state: %v", err)
+
dispatcher.Logger.Warnf("e
rror getting final container state: %v", err)
}
if container.State == dispatch.Locked || container.State == dispatch.Running {
}
if container.State == dispatch.Locked || container.State == dispatch.Running {
-
log.Printf("After %s process termination, container state for %v is %q. U
pdating it to %q",
- *crunchRunCommand,
container.State, uuid
, dispatch.Cancelled)
+
dispatcher.Logger.Warnf("after %q process termination, container state for %v is %q; u
pdating it to %q",
+ *crunchRunCommand,
uuid, container.State
, dispatch.Cancelled)
dispatcher.UpdateState(uuid, dispatch.Cancelled)
}
dispatcher.UpdateState(uuid, dispatch.Cancelled)
}
@@
-183,5
+212,5
@@
func run(dispatcher *dispatch.Dispatcher,
for range status {
}
for range status {
}
-
log.Printf("F
inalized container %v", uuid)
+
dispatcher.Logger.Printf("f
inalized container %v", uuid)
}
}