From: Ward Vandewege Date: Tue, 3 Nov 2020 21:34:16 +0000 (-0500) Subject: 16950: add tests for the costanalyzer. X-Git-Tag: 2.2.0~227^2~6 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/9adb8ea50fcf555c7ec73cb0924c869af2345f86 16950: add tests for the costanalyzer. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- diff --git a/lib/costanalyzer/command.go b/lib/costanalyzer/command.go index 3cca16ea0c..0760b4fd22 100644 --- a/lib/costanalyzer/command.go +++ b/lib/costanalyzer/command.go @@ -22,7 +22,7 @@ func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) { return []byte(entry.Message), nil } -// RunCommand implements the subcommand "deduplication-report ..." +// RunCommand implements the subcommand "costanalyzer ..." func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { var err error logger := ctxlog.New(stderr, "text", "info") diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go index d754e88757..c86e267695 100644 --- a/lib/costanalyzer/costanalyzer.go +++ b/lib/costanalyzer/costanalyzer.go @@ -119,7 +119,9 @@ Usage: provider. - when generating reports for older container requests, the cost data in the Arvados API configuration file may have changed since the container request - was fulfilled. + was fulfilled. This program uses the cost data stored at the time of the + execution of the container, stored in the 'node.json' file in its log + collection. In order to get the data for the uuids supplied, the ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables must be set. @@ -133,7 +135,7 @@ Options: flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.") err := flags.Parse(args) if err == flag.ErrHelp { - exitCode = 0 + exitCode = 1 return } else if err != nil { exitCode = 2 @@ -156,20 +158,21 @@ Options: return } -func ensureDirectory(logger *logrus.Logger, dir string) { +func ensureDirectory(logger *logrus.Logger, dir string) (err error) { statData, err := os.Stat(dir) if os.IsNotExist(err) { err = os.MkdirAll(dir, 0700) if err != nil { logger.Errorf("Error creating directory %s: %s\n", dir, err.Error()) - os.Exit(1) + return } } else { if !statData.IsDir() { logger.Errorf("The path %s is not a directory\n", dir) - os.Exit(1) + return } } + return } func addContainerLine(logger *logrus.Logger, node interface{}, cr Dict, container Dict) (csv string, cost float64) { @@ -245,9 +248,11 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload b } // Load an Arvados object. -func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict) { - - ensureDirectory(logger, path) +func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict, err error) { + err = ensureDirectory(logger, path) + if err != nil { + return + } file := path + "/" + uuid + ".json" @@ -256,9 +261,7 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st if reload { var err error - if strings.Contains(uuid, "-d1hrv-") { - err = arv.Get("pipeline_instances", uuid, nil, &object) - } else if strings.Contains(uuid, "-j7d0g-") { + if strings.Contains(uuid, "-j7d0g-") { err = arv.Get("groups", uuid, nil, &object) } else if strings.Contains(uuid, "-xvhdp-") { err = arv.Get("container_requests", uuid, nil, &object) @@ -268,24 +271,21 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st err = arv.Get("jobs", uuid, nil, &object) } if err != nil { - logger.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err) - os.Exit(1) + logger.Fatalf("Error loading object with UUID %q:\n %s\n", uuid, err) } encoded, err := json.MarshalIndent(object, "", " ") if err != nil { - logger.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err) - os.Exit(1) + logger.Fatalf("Error marshaling object with UUID %q:\n %s\n", uuid, err) } err = ioutil.WriteFile(file, encoded, 0644) if err != nil { - logger.Errorf("Error writing file %s:\n %s\n", file, err) - os.Exit(1) + logger.Fatalf("Error writing file %s:\n %s\n", file, err) } } return } -func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) { +func getNode(arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) { if _, ok := itemMap["log_uuid"]; ok { if itemMap["log_uuid"] == nil { err = errors.New("No log collection") @@ -295,29 +295,28 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva var collection arvados.Collection err = arv.Get("collections", itemMap["log_uuid"].(string), nil, &collection) if err != nil { - logger.Errorf("error getting collection: %s\n", err) + err = fmt.Errorf("Error getting collection: %s", err) return } var fs arvados.CollectionFileSystem fs, err = collection.FileSystem(arv2, kc) if err != nil { - logger.Errorf("error opening collection as filesystem: %s\n", err) + err = fmt.Errorf("Error opening collection as filesystem: %s", err) return } var f http.File f, err = fs.Open("node.json") if err != nil { - logger.Errorf("error opening file in collection: %s\n", err) + err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err) return } var nodeDict Dict - // TODO: checkout io (ioutil?) readall function buf := new(bytes.Buffer) _, err = buf.ReadFrom(f) if err != nil { - logger.Errorf("error reading %q: %s\n", f, err) + err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err) return } contents := buf.String() @@ -325,21 +324,21 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva err = json.Unmarshal([]byte(contents), &nodeDict) if err != nil { - logger.Errorf("error unmarshalling: %s\n", err) + err = fmt.Errorf("Error unmarshalling: %s", err) return } if val, ok := nodeDict["properties"]; ok { var encoded []byte encoded, err = json.MarshalIndent(val, "", " ") if err != nil { - logger.Errorf("error marshalling: %s\n", err) + err = fmt.Errorf("Error marshalling: %s", err) return } // node is type LegacyNodeInfo var newNode LegacyNodeInfo err = json.Unmarshal(encoded, &newNode) if err != nil { - logger.Errorf("error unmarshalling: %s\n", err) + err = fmt.Errorf("Error unmarshalling: %s", err) return } node = newNode @@ -348,7 +347,7 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva var newNode Node err = json.Unmarshal([]byte(contents), &newNode) if err != nil { - logger.Errorf("error unmarshalling: %s\n", err) + err = fmt.Errorf("Error unmarshalling: %s", err) return } node = newNode @@ -361,7 +360,10 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado cost = make(map[string]float64) - project := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) + project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) + if err != nil { + logger.Fatalf("Error loading object %s: %s\n", uuid, err.Error()) + } // arv -f uuid container_request list --filters '[["owner_uuid","=",""],["requesting_container_uuid","=",null]]' @@ -379,7 +381,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado Operand: nil, }, } - err := arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs) + err = arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs) if err != nil { logger.Fatalf("Error querying container_requests: %s\n", err.Error()) } @@ -408,12 +410,18 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado var totalCost float64 // This is a container request, find the container - cr := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) - container := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string)) + cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) + if err != nil { + log.Fatalf("Error loading object %s: %s", uuid, err) + } + container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string)) + if err != nil { + log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err) + } - topNode, err := getNode(logger, arv, arv2, kc, cr) + topNode, err := getNode(arv, arv2, kc, cr) if err != nil { - log.Fatalf("error getting node: %s", err) + log.Fatalf("Error getting node %s: %s\n", cr["uuid"], err) } tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container) csv += tmpCsv @@ -439,9 +447,15 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado for _, item := range items { logger.Info(".") itemMap := item.(map[string]interface{}) - node, _ := getNode(logger, arv, arv2, kc, itemMap) + node, err := getNode(arv, arv2, kc, itemMap) + if err != nil { + log.Fatalf("Error getting node %s: %s\n", itemMap["uuid"], err) + } logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n") - c2 := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string)) + c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string)) + if err != nil { + log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err) + } tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2) cost[itemMap["container_uuid"].(string)] = tmpTotalCost csv += tmpCsv @@ -468,8 +482,11 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log if exitcode != 0 { return } - - ensureDirectory(logger, resultsDir) + err := ensureDirectory(logger, resultsDir) + if err != nil { + exitcode = 3 + return + } // Arvados Client setup arv, err := arvadosclient.MakeArvadosClient() @@ -486,23 +503,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log arv2 := arvados.NewClientFromEnv() cost := make(map[string]float64) - for _, uuid := range uuids { - //csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n" - - if strings.Contains(uuid, "-d1hrv-") { - // This is a pipeline instance, not a job! Find the cwl-runner job. - pi := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) - for _, v := range pi["components"].(map[string]interface{}) { - x := v.(map[string]interface{}) - y := x["job"].(map[string]interface{}) - uuid = y["uuid"].(string) - } - } - - // for projects: - // arv -f uuid container_request list --filters '[["owner_uuid","=",""],["requesting_container_uuid","=",null]]' - if strings.Contains(uuid, "-j7d0g-") { // This is a project (group) for k, v := range handleProject(logger, uuid, arv, arv2, kc, resultsDir) { @@ -528,7 +529,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log if len(cost) == 0 { logger.Info("Nothing to do!\n") - os.Exit(0) + return } var csv string diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go new file mode 100644 index 0000000000..2ef8733b0b --- /dev/null +++ b/lib/costanalyzer/costanalyzer_test.go @@ -0,0 +1,242 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package costanalyzer + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "regexp" + "testing" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/keepclient" + "gopkg.in/check.v1" +) + +func Test(t *testing.T) { + check.TestingT(t) +} + +var _ = check.Suite(&Suite{}) + +type Suite struct{} + +func (s *Suite) TearDownSuite(c *check.C) { + // Undo any changes/additions to the database so they don't affect subsequent tests. + arvadostest.ResetEnv() +} + +func (s *Suite) SetUpSuite(c *check.C) { + arvadostest.StartAPI() + arvadostest.StartKeep(2, true) + + // Get the various arvados, arvadosclient, and keep client objects + ac := arvados.NewClientFromEnv() + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, check.Equals, nil) + arv.ApiToken = arvadostest.ActiveToken + kc, err := keepclient.MakeKeepClient(arv) + c.Assert(err, check.Equals, nil) + + standardE4sV3JSON := `{ + "Name": "Standard_E4s_v3", + "ProviderType": "Standard_E4s_v3", + "VCPUs": 4, + "RAM": 34359738368, + "Scratch": 64000000000, + "IncludedScratch": 64000000000, + "AddedScratch": 0, + "Price": 0.292, + "Preemptible": false +}` + standardD32sV3JSON := `{ + "Name": "Standard_D32s_v3", + "ProviderType": "Standard_D32s_v3", + "VCPUs": 32, + "RAM": 137438953472, + "Scratch": 256000000000, + "IncludedScratch": 256000000000, + "AddedScratch": 0, + "Price": 1.76, + "Preemptible": false +}` + + standardA1V2JSON := `{ + "Name": "a1v2", + "ProviderType": "Standard_A1_v2", + "VCPUs": 1, + "RAM": 2147483648, + "Scratch": 10000000000, + "IncludedScratch": 10000000000, + "AddedScratch": 0, + "Price": 0.043, + "Preemptible": false +}` + + standardA2V2JSON := `{ + "Name": "a2v2", + "ProviderType": "Standard_A2_v2", + "VCPUs": 2, + "RAM": 4294967296, + "Scratch": 20000000000, + "IncludedScratch": 20000000000, + "AddedScratch": 0, + "Price": 0.091, + "Preemptible": false +}` + + // Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON) + + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.DiagnosticsContainerRequest1LogCollectionUUID, standardA1V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, standardA1V2JSON) +} + +func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) { + // Get the CR + var cr arvados.ContainerRequest + err := arv.Get("container_requests", crUUID, arvadosclient.Dict{}, &cr) + c.Assert(err, check.Equals, nil) + c.Assert(cr.LogUUID, check.Equals, logUUID) + + // Get the log collection + var coll arvados.Collection + err = arv.Get("collections", cr.LogUUID, arvadosclient.Dict{}, &coll) + c.Assert(err, check.IsNil) + + // Create a node.json file -- the fixture doesn't actually contain the contents of the collection. + fs, err := coll.FileSystem(ac, kc) + c.Assert(err, check.IsNil) + f, err := fs.OpenFile("node.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777) + c.Assert(err, check.IsNil) + _, err = io.WriteString(f, nodeJSON) + c.Assert(err, check.IsNil) + err = f.Close() + c.Assert(err, check.IsNil) + + // Flush the data to Keep + mtxt, err := fs.MarshalManifest(".") + c.Assert(err, check.IsNil) + c.Assert(mtxt, check.NotNil) + + // Update collection record + err = arv.Update("collections", cr.LogUUID, arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": mtxt}}, &coll) + c.Assert(err, check.IsNil) +} + +func (*Suite) TestUsage(c *check.C) { + var stdout, stderr bytes.Buffer + exitcode := Command.RunCommand("costanalyzer.test", []string{"-help", "-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 1) + c.Check(stdout.String(), check.Equals, "") + c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`) +} + +func (*Suite) TestContainerRequestUUID(c *check.C) { + var stdout, stderr bytes.Buffer + // Run costanalyzer with 1 container request uuid + exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889") +} + +func (*Suite) TestDoubleContainerRequestUUID(c *check.C) { + var stdout, stderr bytes.Buffer + // Run costanalyzer with 2 container request uuids + exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + + uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111") + + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000") + + // Now move both container requests into an existing project, and then re-run + // the analysis with the project uuid. The results should be identical. + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, check.Equals, nil) + + var cr arvados.ContainerRequest + err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr) + c.Assert(err, check.IsNil) + err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID2, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr) + c.Assert(err, check.IsNil) + + // Run costanalyzer with the project uuid + exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + + uuidReport2, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111") + + re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err = ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000") +} + +func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) { + var stdout, stderr bytes.Buffer + // Run costanalyzer with 2 container request uuids + exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00914539") + + uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435") + + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01490377") +} diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go index 9049c73c49..aeb5a47e6d 100644 --- a/sdk/go/arvadostest/fixtures.go +++ b/sdk/go/arvadostest/fixtures.go @@ -44,8 +44,27 @@ const ( RunningContainerUUID = "zzzzz-dz642-runningcontainr" - CompletedContainerUUID = "zzzzz-dz642-compltcontainer" - CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr" + CompletedContainerUUID = "zzzzz-dz642-compltcontainer" + CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr" + CompletedContainerRequestUUID2 = "zzzzz-xvhdp-cr4completedcr2" + + CompletedDiagnosticsContainerRequest1UUID = "zzzzz-xvhdp-diagnostics0001" + CompletedDiagnosticsContainerRequest2UUID = "zzzzz-xvhdp-diagnostics0002" + CompletedDiagnosticsContainer1UUID = "zzzzz-dz642-diagcompreq0001" + CompletedDiagnosticsContainer2UUID = "zzzzz-dz642-diagcompreq0002" + DiagnosticsContainerRequest1LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog1" + DiagnosticsContainerRequest2LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog2" + + CompletedDiagnosticsHasher1ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0001" + CompletedDiagnosticsHasher2ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0002" + CompletedDiagnosticsHasher3ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0003" + CompletedDiagnosticsHasher1ContainerUUID = "zzzzz-dz642-diagcomphasher1" + CompletedDiagnosticsHasher2ContainerUUID = "zzzzz-dz642-diagcomphasher2" + CompletedDiagnosticsHasher3ContainerUUID = "zzzzz-dz642-diagcomphasher3" + + Hasher1LogCollectionUUID = "zzzzz-4zz18-dlogcollhash001" + Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002" + Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003" ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123" ArvadosRepoName = "arvados" @@ -75,7 +94,8 @@ const ( CollectionWithUniqueWordsUUID = "zzzzz-4zz18-mnt690klmb51aud" - LogCollectionUUID = "zzzzz-4zz18-logcollection01" + LogCollectionUUID = "zzzzz-4zz18-logcollection01" + LogCollectionUUID2 = "zzzzz-4zz18-logcollection02" ) // PathologicalManifest : A valid manifest designed to test diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml index 2243d6a447..767f035b88 100644 --- a/services/api/test/fixtures/collections.yml +++ b/services/api/test/fixtures/collections.yml @@ -1043,6 +1043,78 @@ log_collection: manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" name: a real log collection for a completed container +log_collection2: + uuid: zzzzz-4zz18-logcollection02 + current_version_uuid: zzzzz-4zz18-logcollection02 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-10-29T00:51:44.075594000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-10-29T00:51:44.072109000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: another real log collection for a completed container + +diagnostics_request_container_log_collection: + uuid: zzzzz-4zz18-diagcompreqlog1 + current_version_uuid: zzzzz-4zz18-diagcompreqlog1 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:20:44.007557000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:20:44.005381000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: Container log for request zzzzz-xvhdp-diagnostics0001 + +hasher1_log_collection: + uuid: zzzzz-4zz18-dlogcollhash001 + current_version_uuid: zzzzz-4zz18-dlogcollhash001 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:16:55.272606000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:16:55.267006000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: hasher1 log collection + +hasher2_log_collection: + uuid: zzzzz-4zz18-dlogcollhash002 + current_version_uuid: zzzzz-4zz18-dlogcollhash002 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:20:23.547251000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:20:23.545275000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: hasher2 log collection + +hasher3_log_collection: + uuid: zzzzz-4zz18-dlogcollhash003 + current_version_uuid: zzzzz-4zz18-dlogcollhash003 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:20:38.789204000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:20:38.787329000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: hasher3 log collection + +diagnostics_request_container_log_collection2: + uuid: zzzzz-4zz18-diagcompreqlog2 + current_version_uuid: zzzzz-4zz18-diagcompreqlog2 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-03T16:17:53.351593000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-03T16:17:53.346969000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: Container log for request zzzzz-xvhdp-diagnostics0002 + # Test Helper trims the rest of the file # Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml index b3fd6b9dd5..ab0400a678 100644 --- a/services/api/test/fixtures/container_requests.yml +++ b/services/api/test/fixtures/container_requests.yml @@ -115,10 +115,238 @@ completed-older: output_path: test command: ["arvados-cwl-runner", "echo", "hello"] container_uuid: zzzzz-dz642-compltcontainr2 + log_uuid: zzzzz-4zz18-logcollection02 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w runtime_constraints: vcpus: 1 ram: 123 +completed_diagnostics: + name: CWL diagnostics hasher + uuid: zzzzz-xvhdp-diagnostics0001 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 1 + created_at: 2020-11-02T00:03:50.229364000Z + modified_at: 2020-11-02T00:20:44.041122000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-diagcompreqlog1 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 1342177280 + API: true + +completed_diagnostics_hasher1: + name: hasher1 + uuid: zzzzz-xvhdp-diag1hasher0001 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:03:50.229364000Z + modified_at: 2020-11-02T00:20:44.041122000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher1 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher1 + requesting_container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-dlogcollhash001 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + +completed_diagnostics_hasher2: + name: hasher2 + uuid: zzzzz-xvhdp-diag1hasher0002 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:17:07.067464000Z + modified_at: 2020-11-02T00:20:23.557498000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher2 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher2 + requesting_container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-dlogcollhash002 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 2 + ram: 2684354560 + API: true + +completed_diagnostics_hasher3: + name: hasher3 + uuid: zzzzz-xvhdp-diag1hasher0003 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:20:30.960251000Z + modified_at: 2020-11-02T00:20:38.799377000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher3 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher3 + requesting_container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-dlogcollhash003 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + +completed_diagnostics2: + name: Copy of CWL diagnostics hasher + uuid: zzzzz-xvhdp-diagnostics0002 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 1 + created_at: 2020-11-03T15:54:30.098485000Z + modified_at: 2020-11-03T16:17:53.406809000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-diagcompreqlog2 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 1342177280 + API: true + +completed_diagnostics_hasher1_reuse: + name: hasher1 + uuid: zzzzz-xvhdp-diag2hasher0001 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:03:50.229364000Z + modified_at: 2020-11-02T00:20:44.041122000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher1 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher1 + requesting_container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-dlogcollhash001 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + +completed_diagnostics_hasher2_reuse: + name: hasher2 + uuid: zzzzz-xvhdp-diag2hasher0002 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:17:07.067464000Z + modified_at: 2020-11-02T00:20:23.557498000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher2 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher2 + requesting_container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-dlogcollhash002 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 2 + ram: 2684354560 + API: true + +completed_diagnostics_hasher3_reuse: + name: hasher3 + uuid: zzzzz-xvhdp-diag2hasher0003 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:20:30.960251000Z + modified_at: 2020-11-02T00:20:38.799377000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher3 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher3 + requesting_container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-dlogcollhash003 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + requester: uuid: zzzzz-xvhdp-9zacv3o1xw6sxz5 owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml index f18adb5dbd..b7d082771a 100644 --- a/services/api/test/fixtures/containers.yml +++ b/services/api/test/fixtures/containers.yml @@ -126,6 +126,153 @@ completed_older: secret_mounts: {} secret_mounts_md5: 99914b932bd37a50b983c5e7c90ae93b +diagnostics_completed_requester: + uuid: zzzzz-dz642-diagcompreq0001 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:03:50.192697000Z + modified_at: 2020-11-02T00:20:43.987275000Z + started_at: 2020-11-02T00:08:07.186711000Z + finished_at: 2020-11-02T00:20:43.975416000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: 6129e376cb05c942f75a0c36083383e8+244 + output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45 + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 1342177280 + vcpus: 1 + +diagnostics_completed_hasher1: + uuid: zzzzz-dz642-diagcomphasher1 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:08:18.829222000Z + modified_at: 2020-11-02T00:16:55.142023000Z + started_at: 2020-11-02T00:16:52.375871000Z + finished_at: 2020-11-02T00:16:55.105985000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: fed8fb19fe8e3a320c29fed0edab12dd+220 + output: d3a687732e84061f3bae15dc7e313483+62 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 268435456 + vcpus: 1 + +diagnostics_completed_hasher2: + uuid: zzzzz-dz642-diagcomphasher2 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:17:07.026493000Z + modified_at: 2020-11-02T00:20:23.505908000Z + started_at: 2020-11-02T00:20:21.513185000Z + finished_at: 2020-11-02T00:20:23.478317000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: 4fc03b95fc2646b0dec7383dbb7d56d8+221 + output: 6bd770f6cf8f83e7647c602eecfaeeb8+62 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 268435456 + vcpus: 2 + +diagnostics_completed_hasher3: + uuid: zzzzz-dz642-diagcomphasher3 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:20:30.943856000Z + modified_at: 2020-11-02T00:20:38.746541000Z + started_at: 2020-11-02T00:20:36.748957000Z + finished_at: 2020-11-02T00:20:38.732199000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: 1eeaf70de0f65b1346e54c59f09e848d+210 + output: 11b5fdaa380102e760c3eb6de80a9876+62 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 268435456 + vcpus: 1 + +diagnostics_completed_requester2: + uuid: zzzzz-dz642-diagcompreq0002 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 1124295487972526 + created_at: 2020-11-03T15:54:36.504661000Z + modified_at: 2020-11-03T16:17:53.242868000Z + started_at: 2020-11-03T16:09:51.123659000Z + finished_at: 2020-11-03T16:17:53.220358000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: f1933bf5191f576613ea7f65bd0ead53+244 + output: 941b71a57208741ce8742eca62352fb1+123 + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 1342177280 + vcpus: 1 + requester: uuid: zzzzz-dz642-requestingcntnr owner_uuid: zzzzz-tpzed-000000000000000