SHELL ["/bin/bash", "-c"]
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev equivs
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3.8 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3.8-venv python3.8-dev libpam-dev equivs
# Install virtualenv
RUN /usr/bin/pip3 install 'virtualenv<20'
declare -a PYTHON3_BACKPORTS
-PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+PYTHON3_EXECUTABLE=python3
+PYTHON3_VERSION=$($PYTHON3_EXECUTABLE -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
## These defaults are suitable for any Debian-based distribution.
# You can customize them as needed in distro sections below.
debian*)
FORMAT=deb
;;
+ ubuntu1804)
+ FORMAT=deb
+ PYTHON3_EXECUTABLE=python3.8
+ PYTHON3_VERSION=$($PYTHON3_EXECUTABLE -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
+ PYTHON3_PACKAGE=python$PYTHON3_VERSION
+ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
+ ;;
ubuntu*)
FORMAT=deb
;;
ARVADOS_BUILDING_ITERATION=1
fi
- local python=python3
+ local python=$PYTHON3_EXECUTABLE
pip=pip3
PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2021-11-10)
+h2(#main). development main (as of 2022-02-10)
"previous: Upgrading to 2.3.0":#v2_3_0
+h3. Anonymous token changes
+
+The anonymous token configured in @Users.AnonymousUserToken@ must now be 32 characters or longer. This was already the suggestion in the documentation, now it is enforced. The @script/get_anonymous_user_token.rb@ script that was needed to register the anonymous user token in the database has been removed. Registration of the anonymous token is no longer necessary.
+
h3. Preemptible instance types are used automatically, if any are configured
The default behavior for selecting "preemptible instances":{{site.baseurl}}/admin/spot-instances.html has changed. If your configuration lists any instance types with @Preemptible: true@, all child (non-top-level) containers will automatically be scheduled on preemptible instances. To avoid using preemptible instances except when explicitly requested by clients, add @AlwaysUsePreemptibleInstances: false@ in the @Containers@ config section. (Previously, preemptible instance types were never used unless the configuration specified @UsePreemptibleInstances: true@. That flag has been removed.)
# "Introduction":#introduction
# "Configure DNS":#introduction
-# "Configure anonymous user token.yml":#update-config
+# "Configure anonymous user token":#update-config
# "Update nginx configuration":#update-nginx
# "Install keep-web package":#install-packages
# "Start the service":#start-service
h2(#update-config). Configure anonymous user token
-{% assign railscmd = "bin/bundle exec ./script/get_anonymous_user_token.rb --get" %}
-{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token.
-# First, generate a long random string and put it in the @config.yml@ file, in the @AnonymousUserToken@ field.
-# Then, use the following command on the <strong>API server</strong> to register the anonymous user token in the database. {% include 'install_rails_command' %}
+Generate a random string (>= 32 characters long) and put it in the @config.yml@ file, in the @AnonymousUserToken@ field.
<notextile>
<pre><code> Users:
- AnonymousUserToken: <span class="userinput">"{{railsout}}"</span>
+ AnonymousUserToken: <span class="userinput">"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"</span>
</code></pre>
</notextile>
if err != nil {
return err
}
- err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "./script/get_anonymous_user_token.rb")
- if err != nil {
- return err
- }
return nil
}
NewInactiveUserNotificationRecipients: {}
# Set AnonymousUserToken to enable anonymous user access. Populate this
- # field with a long random string. Then run "bundle exec
- # ./script/get_anonymous_user_token.rb" in the directory where your API
- # server is running to record the token in the database.
+ # field with a random string at least 50 characters long.
AnonymousUserToken: ""
# If a new user has an alternate email address (local@domain)
for _, err = range []error{
ldr.checkClusterID(fmt.Sprintf("Clusters.%s", id), id, false),
ldr.checkClusterID(fmt.Sprintf("Clusters.%s.Login.LoginCluster", id), cc.Login.LoginCluster, true),
- ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken),
- ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken),
- ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey),
+ ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken, true),
+ ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken, true),
+ ldr.checkToken(fmt.Sprintf("Clusters.%s.Users.AnonymousUserToken", id), cc.Users.AnonymousUserToken, false),
+ ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey, true),
checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection),
ldr.checkEnum("Containers.LocalKeepLogsToContainerLog", cc.Containers.LocalKeepLogsToContainerLog, "none", "all", "errors"),
ldr.checkEmptyKeepstores(cc),
var acceptableTokenRe = regexp.MustCompile(`^[a-zA-Z0-9]+$`)
var acceptableTokenLength = 32
-func (ldr *Loader) checkToken(label, token string) error {
- if token == "" {
- if ldr.Logger != nil {
- ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength)
+func (ldr *Loader) checkToken(label, token string, mandatory bool) error {
+ if len(token) == 0 {
+ if !mandatory {
+ // when a token is not mandatory, the acceptable length and content is only checked if its length is non-zero
+ return nil
+ } else {
+ if ldr.Logger != nil {
+ ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength)
+ }
}
} else if !acceptableTokenRe.MatchString(token) {
return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label)
return errors.New("not implemented")
}
+func (fw FileWrapper) Snapshot() (*arvados.Subtree, error) {
+ return nil, errors.New("not implemented")
+}
+
+func (fw FileWrapper) Splice(*arvados.Subtree) error {
+ return errors.New("not implemented")
+}
+
func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
if filename == hwImageID+".tar" {
rdr := ioutil.NopCloser(&bytes.Buffer{})
} else {
err = inst.runBash(`
PJS=phantomjs-`+pjsversion+`-linux-x86_64
-wget --progress=dot:giga -O- https://bitbucket.org/ariya/phantomjs/downloads/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
+wget --progress=dot:giga -O- https://cache.arvados.org/$PJS.tar.bz2 | tar -C /var/lib/arvados -xjf -
ln -sf /var/lib/arvados/$PJS/bin/phantomjs /usr/local/bin/
`, stdout, stderr)
if err != nil {
package mount
import (
+ "errors"
"io"
"log"
"os"
}
func (fs *keepFS) errCode(err error) int {
- if os.IsNotExist(err) {
+ if err == nil {
+ return 0
+ }
+ if errors.Is(err, os.ErrNotExist) {
return -fuse.ENOENT
}
- switch err {
- case os.ErrExist:
+ if errors.Is(err, os.ErrExist) {
return -fuse.EEXIST
- case arvados.ErrInvalidArgument:
+ }
+ if errors.Is(err, arvados.ErrInvalidArgument) {
return -fuse.EINVAL
- case arvados.ErrInvalidOperation:
+ }
+ if errors.Is(err, arvados.ErrInvalidOperation) {
return -fuse.ENOSYS
- case arvados.ErrDirectoryNotEmpty:
+ }
+ if errors.Is(err, arvados.ErrDirectoryNotEmpty) {
return -fuse.ENOTEMPTY
- case nil:
- return 0
- default:
- return -fuse.EIO
}
+ return -fuse.EIO
}
func (fs *keepFS) Mkdir(path string, mode uint32) int {
logger.error(str(u"Caught signal {}, exiting.").format(sigcode))
sys.exit(-sigcode)
-def main(args, stdout, stderr, api_client=None, keep_client=None,
+def main(args=sys.argv[1:],
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ api_client=None,
+ keep_client=None,
install_sig_handlers=True):
parser = arg_parser()
metadata = scanobj
sc_result = scandeps(uri, scanobj,
- loadref_fields,
- set(("$include", "$schemas", "location")),
- loadref, urljoin=document_loader.fetcher.urljoin)
+ loadref_fields,
+ set(("$include", "$schemas", "location")),
+ loadref, urljoin=document_loader.fetcher.urljoin,
+ nestdirs=False)
sc = []
uuids = {}
license='Apache 2.0',
packages=find_packages(),
package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml', 'arv-cwl-schema-v1.2.yml']},
- scripts=[
- 'bin/cwl-runner',
- 'bin/arvados-cwl-runner',
- ],
+ entry_points={"console_scripts": ["cwl-runner=arvados_cwl:main", "arvados-cwl-runner=arvados_cwl:main"]},
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20211107152837',
+ 'cwltool==3.1.20220210171524',
'schema-salad==8.2.20211116214159',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
"outstr": "foo woble bar"
tool: 17879-ignore-sbg-fields.cwl
doc: "Test issue 17879 - ignores sbg fields"
+
+- job: chipseq/chip-seq-single.json
+ output: {}
+ tool: chipseq/cwl-packed.json
+ doc: "Test issue 18723 - correctly upload two directories with the same basename"
RUN apt-get update -q && apt-get install -qy --no-install-recommends \
git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
- libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
-
-RUN $pipcmd install -U setuptools six requests
+ libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential ${pythoncmd}-setuptools
ARG sdk
ARG runner
RUN cd /tmp/arvados-python-client-* && $pipcmd install .
RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install networkx==2.2 && $pipcmd install . ; fi
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install . ; fi
RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
# Install dependencies and set up system.
Stat() (os.FileInfo, error)
Truncate(int64) error
Sync() error
+ // Create a snapshot of a file or directory tree, which can
+ // then be spliced onto a different path or a different
+ // collection.
+ Snapshot() (*Subtree, error)
+ // Replace this file or directory with the given snapshot.
+ // The target must be inside a collection: Splice returns an
+ // error if the File is a virtual file or directory like
+ // by_id, a project directory, .arvados#collection,
+ // etc. Splice can replace directories with regular files and
+ // vice versa, except it cannot replace the root directory of
+ // a collection with a regular file.
+ Splice(snapshot *Subtree) error
+}
+
+// A Subtree is a detached part of a filesystem tree that can be
+// spliced into a filesystem via (File)Splice().
+type Subtree struct {
+ inode inode
}
// A FileSystem is an http.Filesystem plus Stat() and support for
Readdir() ([]os.FileInfo, error)
Size() int64
FileInfo() os.FileInfo
+ // Create a snapshot of this node and its descendants.
+ Snapshot() (inode, error)
+ // Replace this node with a copy of the provided snapshot.
+ // Caller may provide the same snapshot to multiple Splice
+ // calls, but must not modify the snapshot concurrently.
+ Splice(inode) error
// Child() performs lookups and updates of named child nodes.
//
return 64
}
+func (*nullnode) Snapshot() (inode, error) {
+ return nil, ErrInvalidOperation
+}
+
+func (*nullnode) Splice(inode) error {
+ return ErrInvalidOperation
+}
+
type treenode struct {
fs FileSystem
parent inode
// supported. Locking inodes from different
// filesystems could deadlock, so we must error out
// now.
- return ErrInvalidArgument
+ return ErrInvalidOperation
}
// To ensure we can test reliably whether we're about to move
func permittedName(name string) bool {
return name != "" && name != "." && name != ".." && !strings.Contains(name, "/")
}
+
+// Snapshot returns a Subtree that's a copy of the given path. It
+// returns an error if the path is not inside a collection.
+func Snapshot(fs FileSystem, path string) (*Subtree, error) {
+ f, err := fs.OpenFile(path, os.O_RDONLY, 0)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return f.Snapshot()
+}
+
+// Splice inserts newsubtree at the indicated target path.
+//
+// Splice returns an error if target is not inside a collection.
+//
+// Splice returns an error if target is the root of a collection and
+// newsubtree is a snapshot of a file.
+func Splice(fs FileSystem, target string, newsubtree *Subtree) error {
+ f, err := fs.OpenFile(target, os.O_WRONLY, 0)
+ if os.IsNotExist(err) {
+ f, err = fs.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0700)
+ }
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ return f.Splice(newsubtree)
+}
return fs.fileSystem.root.(*dirnode).TreeSize()
}
+func (fs *collectionFileSystem) Snapshot() (inode, error) {
+ return fs.fileSystem.root.Snapshot()
+}
+
+func (fs *collectionFileSystem) Splice(r inode) error {
+ return fs.fileSystem.root.Splice(r)
+}
+
// filenodePtr is an offset into a file that is (usually) efficient to
// seek to. Specifically, if filenode.repacked==filenodePtr.repacked
// then
}
}
+func (fn *filenode) Snapshot() (inode, error) {
+ fn.RLock()
+ defer fn.RUnlock()
+ segments := make([]segment, 0, len(fn.segments))
+ for _, seg := range fn.segments {
+ segments = append(segments, seg.Slice(0, seg.Len()))
+ }
+ return &filenode{
+ fileinfo: fn.fileinfo,
+ segments: segments,
+ }, nil
+}
+
+func (fn *filenode) Splice(repl inode) error {
+ repl, err := repl.Snapshot()
+ if err != nil {
+ return err
+ }
+ fn.parent.Lock()
+ defer fn.parent.Unlock()
+ fn.Lock()
+ defer fn.Unlock()
+ _, err = fn.parent.Child(fn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
+ if err != nil {
+ return err
+ }
+ switch repl := repl.(type) {
+ case *dirnode:
+ repl.parent = fn.parent
+ repl.fileinfo.name = fn.fileinfo.name
+ repl.setTreeFS(fn.fs)
+ case *filenode:
+ repl.parent = fn.parent
+ repl.fileinfo.name = fn.fileinfo.name
+ repl.fs = fn.fs
+ default:
+ return fmt.Errorf("cannot splice snapshot containing %T: %w", repl, ErrInvalidArgument)
+ }
+ return nil
+}
+
type dirnode struct {
fs *collectionFileSystem
treenode
return
}
+func (dn *dirnode) Snapshot() (inode, error) {
+ return dn.snapshot()
+}
+
+func (dn *dirnode) snapshot() (*dirnode, error) {
+ dn.RLock()
+ defer dn.RUnlock()
+ snap := &dirnode{
+ treenode: treenode{
+ inodes: make(map[string]inode, len(dn.inodes)),
+ fileinfo: dn.fileinfo,
+ },
+ }
+ for name, child := range dn.inodes {
+ dupchild, err := child.Snapshot()
+ if err != nil {
+ return nil, err
+ }
+ snap.inodes[name] = dupchild
+ dupchild.SetParent(snap, name)
+ }
+ return snap, nil
+}
+
+func (dn *dirnode) Splice(repl inode) error {
+ repl, err := repl.Snapshot()
+ if err != nil {
+ return err
+ }
+ switch repl := repl.(type) {
+ default:
+ return fmt.Errorf("cannot splice snapshot containing %T: %w", repl, ErrInvalidArgument)
+ case *dirnode:
+ dn.Lock()
+ defer dn.Unlock()
+ dn.inodes = repl.inodes
+ dn.setTreeFS(dn.fs)
+ case *filenode:
+ dn.parent.Lock()
+ defer dn.parent.Unlock()
+ removing, err := dn.parent.Child(dn.fileinfo.name, nil)
+ if err != nil {
+ return fmt.Errorf("cannot use Splice to replace a top-level directory with a file: %w", ErrInvalidOperation)
+ } else if removing != dn {
+ // If ../thisdirname is not this dirnode, it
+ // must be an inode that wraps a dirnode, like
+ // a collectionFileSystem or deferrednode.
+ if deferred, ok := removing.(*deferrednode); ok {
+ // More useful to report the type of
+ // the wrapped node rather than just
+ // *deferrednode. (We know the real
+ // inode is already loaded because dn
+ // is inside it.)
+ removing = deferred.realinode()
+ }
+ return fmt.Errorf("cannot use Splice to attach a file at top level of %T: %w", removing, ErrInvalidOperation)
+ }
+ dn.Lock()
+ defer dn.Unlock()
+ _, err = dn.parent.Child(dn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
+ if err != nil {
+ return err
+ }
+ repl.fs = dn.fs
+ }
+ return nil
+}
+
+func (dn *dirnode) setTreeFS(fs *collectionFileSystem) {
+ dn.fs = fs
+ for _, child := range dn.inodes {
+ switch child := child.(type) {
+ case *dirnode:
+ child.setTreeFS(fs)
+ case *filenode:
+ child.fs = fs
+ }
+ }
+}
+
type segment interface {
io.ReaderAt
Len() int
func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
func (dn *deferrednode) MemorySize() int64 { return dn.currentinode().MemorySize() }
+func (dn *deferrednode) Snapshot() (inode, error) { return dn.realinode().Snapshot() }
+func (dn *deferrednode) Splice(repl inode) error { return dn.realinode().Splice(repl) }
// Sync the containing filesystem.
return f.FS().Sync()
}
+
+func (f *filehandle) Snapshot() (*Subtree, error) {
+ if !f.readable {
+ return nil, ErrInvalidOperation
+ }
+ node, err := f.inode.Snapshot()
+ return &Subtree{inode: node}, err
+}
+
+func (f *filehandle) Splice(r *Subtree) error {
+ if !f.writable {
+ return ErrReadOnlyFile
+ }
+ return f.inode.Splice(r.inode)
+}
}
func (*getternode) Child(string, func(inode) (inode, error)) (inode, error) {
- return nil, ErrInvalidArgument
+ return nil, ErrInvalidOperation
}
func (gn *getternode) get() error {
return ln.treenode.Readdir()
}
-// Child rejects (with ErrInvalidArgument) calls to add/replace
+// Child rejects (with ErrInvalidOperation) calls to add/replace
// children, instead calling loadOne when a non-existing child is
// looked up.
func (ln *lookupnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
if replace != nil {
// Let the callback try to delete or replace the
// existing node; if it does, return
- // ErrInvalidArgument.
+ // ErrInvalidOperation.
if tryRepl, err := replace(existing); err != nil {
// Propagate error from callback
return existing, err
} else if tryRepl != existing {
- return existing, ErrInvalidArgument
+ return existing, ErrInvalidOperation
}
}
// Return original error from ln.treenode.Child() (it might be
import (
"bytes"
"encoding/json"
+ "errors"
"io"
"os"
"path/filepath"
s.fs.MountProject("home", "")
_, err := s.fs.OpenFile("/home/A Project/newfilename", os.O_CREATE|os.O_RDWR, 0)
- c.Check(err, check.ErrorMatches, "invalid argument")
+ c.Check(err, ErrorIs, ErrInvalidOperation)
err = s.fs.Mkdir("/home/A Project/newdirname", 0)
- c.Check(err, check.ErrorMatches, "invalid argument")
+ c.Check(err, ErrorIs, ErrInvalidOperation)
err = s.fs.Mkdir("/by_id/newdirname", 0)
- c.Check(err, check.ErrorMatches, "invalid argument")
+ c.Check(err, ErrorIs, ErrInvalidOperation)
err = s.fs.Mkdir("/by_id/"+fixtureAProjectUUID+"/newdirname", 0)
- c.Check(err, check.ErrorMatches, "invalid argument")
+ c.Check(err, ErrorIs, ErrInvalidOperation)
_, err = s.fs.OpenFile("/home/A Project", 0, 0)
c.Check(err, check.IsNil)
}
+
+type errorIsChecker struct {
+ *check.CheckerInfo
+}
+
+var ErrorIs check.Checker = errorIsChecker{
+ &check.CheckerInfo{Name: "ErrorIs", Params: []string{"value", "target"}},
+}
+
+func (checker errorIsChecker) Check(params []interface{}, names []string) (result bool, errStr string) {
+ err, ok := params[0].(error)
+ if !ok {
+ return false, ""
+ }
+ target, ok := params[1].(error)
+ if !ok {
+ return false, ""
+ }
+ return errors.Is(err, target), ""
+}
}
func (fs *customFileSystem) newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error) {
- return nil, ErrInvalidArgument
+ return nil, ErrInvalidOperation
}
func (fs *customFileSystem) mountByID(parent inode, id string) inode {
}
}
-// vdirnode wraps an inode by rejecting (with ErrInvalidArgument)
+// vdirnode wraps an inode by rejecting (with ErrInvalidOperation)
// calls that add/replace children directly, instead calling a
// create() func when a non-existing child is looked up.
//
} else if tryRepl, err := replace(existing); err != nil {
return existing, err
} else if tryRepl != existing {
- return existing, ErrInvalidArgument
+ return existing, ErrInvalidOperation
} else {
return existing, nil
}
package arvados
import (
+ "fmt"
+ "io"
+ "io/ioutil"
"net/http"
"os"
+ "syscall"
"time"
check "gopkg.in/check.v1"
c.Check(names, check.DeepEquals, []string{"baz"})
_, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
- c.Check(err, check.Equals, ErrInvalidArgument)
+ c.Check(err, ErrorIs, ErrInvalidOperation)
err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
- c.Check(err, check.Equals, ErrInvalidArgument)
+ c.Check(err, ErrorIs, ErrInvalidOperation)
err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
- c.Check(err, check.Equals, ErrInvalidArgument)
+ c.Check(err, ErrorIs, ErrInvalidOperation)
_, err = s.fs.Stat("/by_id/beep")
c.Check(err, check.Equals, os.ErrNotExist)
err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
c.Check(err, check.IsNil)
err = s.fs.Rename("/by_id", "/beep")
- c.Check(err, check.Equals, ErrInvalidArgument)
+ c.Check(err, ErrorIs, ErrInvalidOperation)
+}
+
+// Copy subtree from OS src to dst path inside fs. If src is a
+// directory, dst must exist and be a directory.
+func copyFromOS(fs FileSystem, dst, src string) error {
+ inf, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer inf.Close()
+ dirents, err := inf.Readdir(-1)
+ if e, ok := err.(*os.PathError); ok {
+ if e, ok := e.Err.(syscall.Errno); ok {
+ if e == syscall.ENOTDIR {
+ err = syscall.ENOTDIR
+ }
+ }
+ }
+ if err == syscall.ENOTDIR {
+ outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
+ if err != nil {
+ return fmt.Errorf("open %s: %s", dst, err)
+ }
+ defer outf.Close()
+ _, err = io.Copy(outf, inf)
+ if err != nil {
+ return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
+ }
+ err = outf.Close()
+ if err != nil {
+ return err
+ }
+ } else if err != nil {
+ return fmt.Errorf("%s: readdir: %T %s", src, err, err)
+ } else {
+ {
+ d, err := fs.Open(dst)
+ if err != nil {
+ return fmt.Errorf("opendir(%s): %s", dst, err)
+ }
+ d.Close()
+ }
+ for _, ent := range dirents {
+ if ent.Name() == "." || ent.Name() == ".." {
+ continue
+ }
+ dstname := dst + "/" + ent.Name()
+ if ent.IsDir() {
+ err = fs.Mkdir(dstname, 0700)
+ if err != nil {
+ return fmt.Errorf("mkdir %s: %s", dstname, err)
+ }
+ }
+ err = copyFromOS(fs, dstname, src+"/"+ent.Name())
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
+ s.fs.MountProject("home", "")
+ thisfile, err := ioutil.ReadFile("fs_site_test.go")
+ c.Assert(err, check.IsNil)
+
+ var src1 Collection
+ err = s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "name": "TestSnapshotSplice src1",
+ "owner_uuid": fixtureAProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
+ err = s.fs.Sync()
+ c.Assert(err, check.IsNil)
+ err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
+ c.Assert(err, check.IsNil)
+
+ var src2 Collection
+ err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "name": "TestSnapshotSplice src2",
+ "owner_uuid": fixtureAProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
+ err = s.fs.Sync()
+ c.Assert(err, check.IsNil)
+ err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
+ c.Assert(err, check.IsNil)
+
+ var dst Collection
+ err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "name": "TestSnapshotSplice dst",
+ "owner_uuid": fixtureAProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
+ err = s.fs.Sync()
+ c.Assert(err, check.IsNil)
+
+ dstPath := "/home/A Project/TestSnapshotSplice dst"
+ err = copyFromOS(s.fs, dstPath, "..") // arvados.git/sdk/go
+ c.Assert(err, check.IsNil)
+
+ // Snapshot directory
+ snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
+ c.Check(err, check.IsNil)
+ // Attach same snapshot twice, at paths that didn't exist before
+ err = Splice(s.fs, dstPath+"/ctxlog-copy", snap1)
+ c.Check(err, check.IsNil)
+ err = Splice(s.fs, dstPath+"/ctxlog-copy2", snap1)
+ c.Check(err, check.IsNil)
+ // Splicing a snapshot twice results in two independent copies
+ err = s.fs.Rename(dstPath+"/ctxlog-copy2/log.go", dstPath+"/ctxlog-copy/log2.go")
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open(dstPath + "/ctxlog-copy2/log.go")
+ c.Check(err, check.Equals, os.ErrNotExist)
+ f, err := s.fs.Open(dstPath + "/ctxlog-copy/log.go")
+ if c.Check(err, check.IsNil) {
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Not(check.Equals), "")
+ f.Close()
+ }
+
+ // Snapshot regular file
+ snapFile, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/arvados/fs_site_test.go")
+ c.Check(err, check.IsNil)
+ // Replace dir with file
+ err = Splice(s.fs, dstPath+"/ctxlog-copy2", snapFile)
+ c.Check(err, check.IsNil)
+ if f, err := s.fs.Open(dstPath + "/ctxlog-copy2"); c.Check(err, check.IsNil) {
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, string(thisfile))
+ }
+
+ // Cannot splice a file onto a collection root, or anywhere
+ // outside a collection
+ for _, badpath := range []string{
+ dstPath,
+ "/home/A Project/newnodename",
+ "/home/A Project",
+ "/home/newnodename",
+ "/home",
+ "/newnodename",
+ } {
+ err = Splice(s.fs, badpath, snapFile)
+ c.Check(err, check.NotNil)
+ c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
+ if badpath == dstPath {
+ c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %s", badpath))
+ continue
+ }
+ err = Splice(s.fs, badpath, snap1)
+ c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
+ }
+
+ // Destination cannot have trailing slash
+ for _, badpath := range []string{
+ dstPath + "/ctxlog/",
+ dstPath + "/",
+ "/home/A Project/",
+ "/home/",
+ "/",
+ "",
+ } {
+ err = Splice(s.fs, badpath, snap1)
+ c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
+ err = Splice(s.fs, badpath, snapFile)
+ c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
+ }
+
+ // Destination's parent must already exist
+ for _, badpath := range []string{
+ dstPath + "/newdirname/",
+ dstPath + "/newdirname/foobar",
+ "/foo/bar",
+ } {
+ err = Splice(s.fs, badpath, snap1)
+ c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
+ err = Splice(s.fs, badpath, snapFile)
+ c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %s", badpath))
+ }
+
+ snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
+ c.Check(err, check.IsNil)
+ err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
+ c.Check(err, check.IsNil)
+
+ // Snapshot entire collection, splice into same collection at
+ // a new path, remove file from original location, verify
+ // spliced content survives
+ snapDst, err := Snapshot(s.fs, dstPath+"")
+ c.Check(err, check.IsNil)
+ err = Splice(s.fs, dstPath+"", snapDst)
+ c.Check(err, check.IsNil)
+ err = Splice(s.fs, dstPath+"/copy1", snapDst)
+ c.Check(err, check.IsNil)
+ err = Splice(s.fs, dstPath+"/copy2", snapDst)
+ c.Check(err, check.IsNil)
+ err = s.fs.RemoveAll(dstPath + "/arvados/fs_site_test.go")
+ c.Check(err, check.IsNil)
+ err = s.fs.RemoveAll(dstPath + "/arvados")
+ c.Check(err, check.IsNil)
+ _, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
+ c.Check(err, check.Equals, os.ErrNotExist)
+ f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
+ c.Check(err, check.IsNil)
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, string(thisfile))
}
clnt
end
+ def self.check_anonymous_user_token token
+ case token[0..2]
+ when 'v2/'
+ _, token_uuid, secret, optional = token.split('/')
+ unless token_uuid.andand.length == 27 && secret.andand.length.andand > 0 &&
+ token_uuid == Rails.configuration.ClusterID+"-gj3su-anonymouspublic"
+ # invalid v2 token, or v2 token for another user
+ return nil
+ end
+ else
+ # v1 token
+ secret = token
+ end
+
+ # The anonymous token content and minimum length is verified in lib/config
+ if secret.length >= 0 && secret == Rails.configuration.Users.AnonymousUserToken
+ return ApiClientAuthorization.new(user: User.find_by_uuid(anonymous_user_uuid),
+ uuid: Rails.configuration.ClusterID+"-gj3su-anonymouspublic",
+ api_token: token,
+ api_client: anonymous_user_token_api_client)
+ else
+ return nil
+ end
+ end
+
def self.check_system_root_token token
if token == Rails.configuration.SystemRootToken
return ApiClientAuthorization.new(user: User.find_by_uuid(system_user_uuid),
return nil if token.nil? or token.empty?
remote ||= Rails.configuration.ClusterID
+ auth = self.check_anonymous_user_token(token)
+ if !auth.nil?
+ return auth
+ end
+
auth = self.check_system_root_token(token)
if !auth.nil?
return auth
anonymous_group
anonymous_group_read_permission
anonymous_user
+ anonymous_user_token_api_client
system_root_token_api_client
public_project_group
public_project_read_permission
end
end
+ def anonymous_user_token_api_client
+ $anonymous_user_token_api_client = check_cache $anonymous_user_token_api_client do
+ act_as_system_user do
+ ActiveRecord::Base.transaction do
+ ApiClient.find_or_create_by!(is_trusted: false, url_prefix: "", name: "AnonymousUserToken")
+ end
+ end
+ end
+ end
+
def system_root_token_api_client
$system_root_token_api_client = check_cache $system_root_token_api_client do
act_as_system_user do
+++ /dev/null
-#!/usr/bin/env ruby
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# Get or Create an anonymous user token.
-# If get option is used, an existing anonymous user token is returned. If none exist, one is created.
-# If the get option is omitted, a new token is created and returned.
-
-require 'optimist'
-
-opts = Optimist::options do
- banner ''
- banner "Usage: get_anonymous_user_token "
- banner ''
- opt :get, <<-eos
-Get an existing anonymous user token. If no such token exists \
-or if this option is omitted, a new token is created and returned.
- eos
- opt :token, "token to create (optional)", :type => :string
-end
-
-get_existing = opts[:get]
-supplied_token = opts[:token]
-
-require File.dirname(__FILE__) + '/../config/environment'
-
-include ApplicationHelper
-act_as_system_user
-
-def create_api_client_auth(supplied_token=nil)
- supplied_token = Rails.configuration.Users["AnonymousUserToken"]
-
- if supplied_token.nil? or supplied_token.empty?
- puts "Users.AnonymousUserToken is empty. Destroying tokens that belong to anonymous."
- # Token is empty. Destroy any anonymous tokens.
- ApiClientAuthorization.where(user: anonymous_user).destroy_all
- return nil
- end
-
- attr = {user: anonymous_user,
- api_client_id: 0,
- scopes: ['GET /']}
-
- secret = supplied_token
-
- if supplied_token[0..2] == 'v2/'
- _, token_uuid, secret, optional = supplied_token.split('/')
- if token_uuid[0..4] != Rails.configuration.ClusterID
- # Belongs to a different cluster.
- puts supplied_token
- return nil
- end
- attr[:uuid] = token_uuid
- end
-
- attr[:api_token] = secret
-
- api_client_auth = ApiClientAuthorization.where(attr).first
- if !api_client_auth
- # The anonymous user token should never expire but we are not allowed to
- # set :expires_at to nil, so we set it to 1000 years in the future.
- attr[:expires_at] = Time.now + 1000.years
- api_client_auth = ApiClientAuthorization.create!(attr)
- end
- api_client_auth
-end
-
-if get_existing
- api_client_auth = ApiClientAuthorization.
- where('user_id=?', anonymous_user.id.to_i).
- where('expires_at>?', Time.now).
- select { |auth| auth.scopes == ['GET /'] }.
- first
-end
-
-# either not a get or no api_client_auth was found
-if !api_client_auth
- api_client_auth = create_api_client_auth(supplied_token)
-end
-
-# print it to the console
-if api_client_auth
- puts "v2/#{api_client_auth.uuid}/#{api_client_auth.api_token}"
-end
return true
}
err = fs.Mkdir(dir, 0755)
- if err == arvados.ErrInvalidArgument {
+ if errors.Is(err, arvados.ErrInvalidArgument) || errors.Is(err, arvados.ErrInvalidOperation) {
// Cannot create a directory
// here.
err = fmt.Errorf("mkdir %q failed: %w", dir, err)
err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
- c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
+ c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid (argument|operation)`)
_, err = bucket.GetReader(trial.path)
c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
fi
if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
- bin/bundle exec rake db:setup
+ flock $GEMLOCK bin/bundle exec rake db:setup
touch $ARVADOS_CONTAINER_PATH/api_database_setup
fi
if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
- superuser_tok=$(bin/bundle exec ./script/create_superuser_token.rb)
+ superuser_tok=$(flock $GEMLOCK bin/bundle exec ./script/create_superuser_token.rb)
echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
fi
rm -rf tmp
mkdir -p tmp/cache
-bin/bundle exec rake db:migrate
+flock $GEMLOCK bin/bundle exec rake db:migrate
export R_LIBS=/var/lib/Rlibs
export HOME=$(getent passwd arvbox | cut -d: -f6)
export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox
+GEMLOCK=/var/lib/arvados/lib/ruby/gems/gems.lock
defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }')
dockerip=$(/sbin/ip route | grep default | awk '{ print $3 }')
fi
run_bundler() {
- GEMLOCK=/var/lib/arvados/lib/ruby/gems/gems.lock
flock $GEMLOCK /var/lib/arvados/bin/gem install --no-document bundler:$BUNDLER_VERSION
if test -f Gemfile.lock ; then
frozen=--frozen
fi
cd /usr/src/arvados/doc
-bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
+flock $GEMLOCK bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
EOF
while true ; do
- bundle exec script/arvados-git-sync.rb $RAILS_ENV
+ flock $GEMLOCK bundle exec script/arvados-git-sync.rb $RAILS_ENV
sleep 120
done
$RAILS_ENV:
keep_web_url: https://example.com/c=%{uuid_or_pdh}
EOF
- RAILS_GROUPS=assets bin/bundle exec rake npm:install
+ RAILS_GROUPS=assets flock $GEMLOCK bin/bundle exec rake npm:install
rm config/application.yml
exit
fi
secret_token=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
-RAILS_GROUPS=assets bin/bundle exec rake npm:install
-bin/bundle exec rake assets:precompile
+RAILS_GROUPS=assets flock $GEMLOCK bin/bundle exec rake npm:install
+flock $GEMLOCK bin/bundle exec rake assets:precompile
projects / arvados.git / commitdiff