-package main
+// Copyright (C) The Lightning Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package lightning
import (
+ "bufio"
"context"
"encoding/json"
"errors"
"sync"
"time"
+ "git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/klauspost/pgzip"
log "github.com/sirupsen/logrus"
"golang.org/x/crypto/blake2b"
"golang.org/x/net/websocket"
"runtime_constraints": rc,
"priority": runner.Priority,
"state": arvados.ContainerRequestStateCommitted,
+ "scheduling_parameters": arvados.SchedulingParameters{
+ Preemptible: true,
+ Partitions: []string{},
+ },
+ "environment": map[string]string{
+ "GOMAXPROCS": fmt.Sprintf("%d", rc.VCPUs),
+ },
},
})
if err != nil {
subscribedUUID := ""
defer func() {
if subscribedUUID != "" {
+ log.Printf("unsubscribe container UUID: %s", subscribedUUID)
client.Unsubscribe(logch, subscribedUUID)
}
}()
fmt.Fprint(os.Stderr, neednewline)
neednewline = ""
if subscribedUUID != "" {
+ log.Printf("unsubscribe container UUID: %s", subscribedUUID)
client.Unsubscribe(logch, subscribedUUID)
}
+ log.Printf("subscribe container UUID: %s", cr.ContainerUUID)
client.Subscribe(logch, cr.ContainerUUID)
subscribedUUID = cr.ContainerUUID
}
return "", err
}
b2 := blake2b.Sum256(exe)
- cname := fmt.Sprintf("lightning-%x", b2)
+ cname := "lightning " + cmd.Version.String() // must build with "make", not just "go install"
var existing arvados.CollectionList
err = runner.Client.RequestAndDecode(&existing, "GET", "arvados/v1/collections", nil, arvados.ListOptions{
Limit: 1,
Filters: []arvados.Filter{
{Attr: "name", Operator: "=", Operand: cname},
{Attr: "owner_uuid", Operator: "=", Operand: runner.ProjectUUID},
+ {Attr: "properties.blake2b", Operator: "=", Operand: fmt.Sprintf("%x", b2)},
},
})
if err != nil {
return "", err
}
if len(existing.Items) > 0 {
- uuid := existing.Items[0].UUID
- log.Printf("using lightning binary in existing collection %s (name is %q; did not verify whether content matches)", uuid, cname)
- return uuid, nil
+ coll := existing.Items[0]
+ log.Printf("using lightning binary in existing collection %s (name is %q, hash is %q; did not verify whether content matches)", coll.UUID, cname, coll.Properties["blake2b"])
+ return coll.UUID, nil
}
log.Printf("writing lightning binary to new collection %q", cname)
ac, err := arvadosclient.New(runner.Client)
"owner_uuid": runner.ProjectUUID,
"manifest_text": mtxt,
"name": cname,
+ "properties": map[string]interface{}{
+ "blake2b": fmt.Sprintf("%x", b2),
+ },
},
})
if err != nil {
return coll.UUID, nil
}
-var arvadosClientFromEnv = arvados.NewClientFromEnv()
+// zopen returns a reader for the given file, using the arvados API
+// instead of arv-mount/fuse where applicable, and transparently
+// decompressing the input if fnm ends with ".gz".
+func zopen(fnm string) (io.ReadCloser, error) {
+ f, err := open(fnm)
+ if err != nil || !strings.HasSuffix(fnm, ".gz") {
+ return f, err
+ }
+ rdr, err := pgzip.NewReader(bufio.NewReaderSize(f, 4*1024*1024))
+ if err != nil {
+ f.Close()
+ return nil, err
+ }
+ return gzipr{rdr, f}, nil
+}
+
+// gzipr wraps a ReadCloser and a Closer, presenting a single Close()
+// method that closes both wrapped objects.
+type gzipr struct {
+ io.ReadCloser
+ io.Closer
+}
+
+func (gr gzipr) Close() error {
+ e1 := gr.ReadCloser.Close()
+ e2 := gr.Closer.Close()
+ if e1 != nil {
+ return e1
+ }
+ return e2
+}
+
+var (
+ arvadosClientFromEnv = arvados.NewClientFromEnv()
+ keepClient *keepclient.KeepClient
+ siteFS arvados.CustomFileSystem
+ siteFSMtx sync.Mutex
+)
+
+type file interface {
+ io.ReadCloser
+ Readdir(n int) ([]os.FileInfo, error)
+}
-func open(fnm string) (io.ReadCloser, error) {
+func open(fnm string) (file, error) {
if os.Getenv("ARVADOS_API_HOST") == "" {
return os.Open(fnm)
}
return os.Open(fnm)
}
uuid := m[2]
- mnt := "/mnt/" + uuid + "/"
- if !strings.HasPrefix(fnm, mnt) {
+ mnt := "/mnt/" + uuid
+ if fnm != mnt && !strings.HasPrefix(fnm, mnt+"/") {
return os.Open(fnm)
}
- log.Infof("reading %q from %s using Arvados client library", fnm[len(mnt):], uuid)
- ac, err := arvadosclient.New(arvadosClientFromEnv)
- if err != nil {
- return nil, err
- }
- ac.Client = arvados.DefaultSecureClient
- kc := keepclient.New(ac)
- // Don't use keepclient's default short timeouts.
- kc.HTTPClient = arvados.DefaultSecureClient
- // Don't cache more than one block for this file.
- kc.BlockCache = &keepclient.BlockCache{MaxBlocks: 1}
-
- var coll arvados.Collection
- err = arvadosClientFromEnv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+uuid, nil, arvados.GetOptions{Select: []string{"uuid", "manifest_text"}})
- if err != nil {
- return nil, err
- }
- fs, err := coll.FileSystem(arvadosClientFromEnv, kc)
- if err != nil {
- return nil, err
- }
- return fs.Open(fnm[len(mnt):])
+ siteFSMtx.Lock()
+ defer siteFSMtx.Unlock()
+ if siteFS == nil {
+ log.Info("setting up Arvados client")
+ ac, err := arvadosclient.New(arvadosClientFromEnv)
+ if err != nil {
+ return nil, err
+ }
+ ac.Client = arvados.DefaultSecureClient
+ keepClient = keepclient.New(ac)
+ // Don't use keepclient's default short timeouts.
+ keepClient.HTTPClient = arvados.DefaultSecureClient
+ keepClient.BlockCache = &keepclient.BlockCache{MaxBlocks: 4}
+ siteFS = arvadosClientFromEnv.SiteFileSystem(keepClient)
+ } else {
+ keepClient.BlockCache.MaxBlocks++
+ }
+
+ log.Infof("reading %q from %s using Arvados client", fnm[len(mnt):], uuid)
+ return siteFS.Open("by_id/" + uuid + fnm[len(mnt):])
}
projects / lightning.git / blobdiff