Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>
-[//]: # Copyright (C) The Arvados Authors. All rights reserved.
-[//]: #
-[//]: # SPDX-License-Identifier: CC-BY-SA-3.0
+[comment]: # (Copyright © The Arvados Authors. All rights reserved.)
+[comment]: # ()
+[comment]: # (SPDX-License-Identifier: CC-BY-SA-3.0)
[Arvados](https://arvados.org) is a free software distributed computing platform
for bioinformatics, data science, and high throughput analysis of massive data
## Community
-The [#arvados](irc://irc.oftc.net:6667/#arvados IRC) (Internet Relay Chat)
+The [#arvados](irc://irc.oftc.net:6667/#arvados) IRC (Internet Relay Chat)
channel at the
[Open and Free Technology Community (irc.oftc.net)](http://www.oftc.net/oftc/)
is available for live discussion and support. You can use a traditional IRC
development. The
[Arvados developer mailing list](http://lists.arvados.org/mailman/listinfo/arvados-dev)
is a forum for more technical discussion, intended for developers and
-contributers to Arvados.
+contributors to Arvados.
## Development
components can be found on the
["Hacking Arvados" page of the Arvados wiki](https://dev.arvados.org/projects/arvados/wiki/Hacking).
+## Contributing
+
+When making a pull request, please ensure *every git commit message* includes a one-line [Developer Certificate of Origin](https://dev.arvados.org/projects/arvados/wiki/Developer_Certificate_Of_Origin). If you have already made commits without it, fix them with `git commit --amend` or `git rebase`.
+
## Licensing
Arvados is Free Software. See COPYING for information about Arvados Free
IFS=$'\n' read -a ignores -r -d $'\000' <.licenseignore || true
result=0
-git ls-files -z ${@} | \
- while read -rd $'\000' fnm
+
+coproc git ls-files -z ${@} </dev/null
+while read -rd $'\000' fnm
+do
+ grepAfter=2
+ grepBefore=0
+ cs=
+ cc=
+ ce=
+ fixer=
+ if [[ ! -f ${fnm} ]] || [[ -L ${fnm} ]] || [[ ! -s ${fnm} ]]
+ then
+ continue
+ fi
+
+ ignore=
+ for pattern in "${ignores[@]}"
do
- grepAfter=2
- grepBefore=0
- cs=
- cc=
- ce=
- fixer=
- if [[ ! -f ${fnm} ]] || [[ -L ${fnm} ]] || [[ ! -s ${fnm} ]]
+ if [[ ${fnm} == ${pattern} ]]
then
- continue
+ ignore=1
fi
+ done
+ if [[ ${ignore} = 1 ]]; then continue; fi
- ignore=
- for pattern in "${ignores[@]}"
- do
- if [[ ${fnm} == ${pattern} ]]
+ case ${fnm} in
+ Makefile | */Makefile \
+ | *.dockerfile | */Dockerfile.* | */Dockerfile | *.dockerignore \
+ | */MANIFEST.in | */fuse.conf | */gitolite.rc \
+ | *.pl | *.pm | *.PL \
+ | *.rb | *.rb.example | *.rake | *.ru \
+ | *.gemspec | */Gemfile | */Rakefile \
+ | services/login-sync/bin/* \
+ | sdk/cli/bin/* \
+ | *.py \
+ | sdk/python/bin/arv-* \
+ | sdk/cwl/bin/* \
+ | services/nodemanager/bin/* \
+ | services/fuse/bin/* \
+ | tools/crunchstat-summary/bin/* \
+ | crunch_scripts/* \
+ | *.yaml | *.yml | *.yml.example | *.cwl \
+ | *.sh | *.service \
+ | */run | */run-service | */restart-dns-server \
+ | */nodemanager/doc/*.cfg \
+ | */nodemanager/tests/fake*.cfg.template \
+ | */nginx.conf \
+ | build/build.list)
+ fixer=fixer
+ cc="#"
+ ;;
+ *.md)
+ fixer=fixer
+ cc="[//]: #"
+ ;;
+ *.rst)
+ fixer=fixer
+ cc=".."
+ ;;
+ *.erb)
+ fixer=fixer
+ cs="<%# "
+ cc=""
+ ce=" %>"
+ ;;
+ *.liquid)
+ fixer=fixer
+ cs=$'{% comment %}\n'
+ cc=""
+ ce=$'\n{% endcomment %}'
+ grepAfter=3
+ grepBefore=1
+ ;;
+ *.textile)
+ fixer=fixer
+ cs="###. "
+ cc="...."
+ ce=
+ ;;
+ *.css)
+ fixer=fixer
+ cs="/* "
+ cc=""
+ ce=" */"
+ ;;
+ *.coffee)
+ fixer=fixer
+ cs="### "
+ cc=""
+ ce=" ###"
+ ;;
+ *.go | *.scss | *.java | *.js)
+ fixer=fixer
+ cc="//"
+ ;;
+ *.sql)
+ fixer=fixer
+ cc="--"
+ ;;
+ *.html | *.svg)
+ fixer=fixer
+ cs="<!-- "
+ cc=""
+ ce=" -->"
+ ;;
+ *)
+ cc="#"
+ hashbang=$(head -n1 ${fnm})
+ if [[ ${hashbang} = "#!/bin/sh" ]] || [[ ${hashbang} = "#!/bin/bash" ]]
then
- ignore=1
- fi
- done
- if [[ ${ignore} = 1 ]]; then continue; fi
-
- case ${fnm} in
- Makefile | */Makefile \
- | *.dockerfile | */Dockerfile.* | */Dockerfile | *.dockerignore \
- | */MANIFEST.in | */fuse.conf | */gitolite.rc \
- | *.pl | *.pm | *.PL \
- | *.rb | *.rb.example | *.rake | *.ru \
- | *.gemspec | */Gemfile | */Rakefile \
- | services/login-sync/bin/* \
- | sdk/cli/bin/* \
- | *.py \
- | sdk/python/bin/arv-* \
- | sdk/cwl/bin/* \
- | services/nodemanager/bin/* \
- | services/fuse/bin/* \
- | tools/crunchstat-summary/bin/* \
- | crunch_scripts/* \
- | *.yaml | *.yml | *.yml.example | *.cwl \
- | *.sh | *.service \
- | */run | */run-service | */restart-dns-server \
- | */nodemanager/doc/*.cfg \
- | */nodemanager/tests/fake*.cfg.template \
- | */nginx.conf \
- | build/build.list)
- fixer=fixer
- cc="#"
- ;;
- *.md)
- fixer=fixer
- cc="[//]: #"
- ;;
- *.rst)
- fixer=fixer
- cc=".."
- ;;
- *.erb)
- fixer=fixer
- cs="<%# "
- cc=""
- ce=" %>"
- ;;
- *.liquid)
- fixer=fixer
- cs=$'{% comment %}\n'
- cc=""
- ce=$'\n{% endcomment %}'
- grepAfter=3
- grepBefore=1
- ;;
- *.textile)
- fixer=fixer
- cs="###. "
- cc="...."
- ce=
- ;;
- *.css)
- fixer=fixer
- cs="/* "
- cc=""
- ce=" */"
- ;;
- *.coffee)
fixer=fixer
- cs="### "
- cc=""
- ce=" ###"
- ;;
- *.go | *.scss | *.java | *.js)
- fixer=fixer
- cc="//"
- ;;
- *.sql)
- fixer=fixer
- cc="--"
- ;;
- *.html | *.svg)
- fixer=fixer
- cs="<!-- "
- cc=""
- ce=" -->"
- ;;
- *)
- cc="#"
- hashbang=$(head -n1 ${fnm})
- if [[ ${hashbang} = "#!/bin/sh" ]] || [[ ${hashbang} = "#!/bin/bash" ]]
- then
- fixer=fixer
- fi
- ;;
- esac
- wantGPL="${cs:-${cc}${cc:+ }}Copyright (C) The Arvados Authors. All rights reserved.
+ fi
+ ;;
+ esac
+ wantGPL="${cs:-${cc}${cc:+ }}Copyright (C) The Arvados Authors. All rights reserved.
${cc}
${cc}${cc:+ }SPDX-License-Identifier: AGPL-3.0${ce}"
- wantApache="${cs:-${cc}${cc:+ }}Copyright (C) The Arvados Authors. All rights reserved.
+ wantApache="${cs:-${cc}${cc:+ }}Copyright (C) The Arvados Authors. All rights reserved.
${cc}
${cc}${cc:+ }SPDX-License-Identifier: Apache-2.0${ce}"
- wantBYSA="${cs:-${cc}${cc:+ }}Copyright (C) The Arvados Authors. All rights reserved.
+ wantBYSA="${cs:-${cc}${cc:+ }}Copyright (C) The Arvados Authors. All rights reserved.
${cc}
${cc}${cc:+ }SPDX-License-Identifier: CC-BY-SA-3.0${ce}"
- found=$(head -n20 "$fnm" | egrep -A${grepAfter} -B${grepBefore} 'Copyright.*Arvados' || true)
- case ${fnm} in
- Makefile | build/* | lib/* | tools/* | apps/* | services/* | sdk/cli/bin/crunch-job)
- want=${wantGPL}
- ;;
- crunch_scripts/* | backports/* | docker/* | sdk/*)
- want=${wantApache}
- ;;
- README.md | doc/*)
- want=${wantBYSA}
- ;;
- *)
- want=
- ;;
- esac
- case "$found" in
- "$wantGPL")
- ;;
- "$wantApache")
- ;;
- "$wantBYSA")
- ;;
- "")
- if [[ -z ${found} ]] && [[ -n ${want} ]] && [[ $fix = true ]] && [[ $fixer != "" ]]
- then
- ${fixer} ${fnm}
- else
- echo "missing copyright notice: $fnm"
- result=1
- fi
- ;;
- *)
- echo "nonstandard copyright notice: $fnm '${found}'"
+ wantBYSAmd="[comment]: # (Copyright © The Arvados Authors. All rights reserved.)
+[comment]: # ()
+[comment]: # (SPDX-License-Identifier: CC-BY-SA-3.0)"
+ found=$(head -n20 "$fnm" | egrep -A${grepAfter} -B${grepBefore} 'Copyright.*Arvados' || true)
+ case ${fnm} in
+ Makefile | build/* | lib/* | tools/* | apps/* | services/* | sdk/cli/bin/crunch-job)
+ want=${wantGPL}
+ ;;
+ crunch_scripts/* | backports/* | docker/* | sdk/*)
+ want=${wantApache}
+ ;;
+ doc/*)
+ want=${wantBYSA}
+ ;;
+ README.md)
+ want=${wantBYSAmd}
+ ;;
+ *)
+ want=
+ ;;
+ esac
+ case "$found" in
+ "$wantGPL")
+ ;;
+ "$wantApache")
+ ;;
+ "$wantBYSA")
+ ;;
+ "$wantBYSAmd")
+ ;;
+ "")
+ if [[ -z ${found} ]] && [[ -n ${want} ]] && [[ $fix = true ]] && [[ $fixer != "" ]]
+ then
+ ${fixer} ${fnm}
+ else
+ echo "missing copyright notice: $fnm"
result=1
- ;;
- esac
- done
+ fi
+ ;;
+ *)
+ echo "nonstandard copyright notice: $fnm '${found}'"
+ result=1
+ ;;
+ esac
+done <&${COPROC[0]}
exit $result
echo >&2 "usage: $0 [options]"
echo >&2
echo >&2 "$0 options:"
+ echo >&2 " -t, --tags version tag for docker"
echo >&2 " -u, --upload Upload the images (docker push)"
echo >&2 " --no-cache Don't use build cache"
echo >&2 " -h, --help Display this help and exit"
echo >&2
echo >&2 " If no options are given, just builds the images."
}
-
upload=false
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
exit 1
;;
*)
- echo "WARNING: --tags is deprecated and doesn't do anything";
+ version_tag="$2";
shift 2
;;
esac
# -f flag removed in Docker 1.12
FORCE=-f
fi
-
-docker tag $FORCE arvados/jobs:$cwl_runner_version arvados/jobs:latest
+if ! [[ -z "$version_tag" ]]; then
+ docker tag $FORCE arvados/jobs:$cwl_runner_version arvados/jobs:"$version_tag"
+else
+ docker tag $FORCE arvados/jobs:$cwl_runner_version arvados/jobs:latest
+fi
ECODE=$?
## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
-
- docker_push arvados/jobs:$cwl_runner_version
- docker_push arvados/jobs:latest
+ if ! [[ -z "$version_tag" ]]; then
+ docker_push arvados/jobs:"$version_tag"
+ else
+ docker_push arvados/jobs:$cwl_runner_version
+ docker_push arvados/jobs:latest
+ fi
title "upload arvados images finished (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set (`timer`)"
if test -z "$packages" ; then
packages="arvados-api-server
+ arvados-docker-cleaner
arvados-git-httpd
arvados-node-manager
arvados-src
python-arvados-fuse
python-arvados-python-client
python-arvados-cwl-runner"
-
- if [[ $TARGET != centos7 ]]; then
- packages="$packages arvados-docker-cleaner"
- fi
fi
FINAL_EXITCODE=0
fi
local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")"
+ local python=""
case "$PACKAGE_TYPE" in
python)
# All Arvados Python2 packages depend on Python 2.7.
# Make sure we build with that for consistency.
+ python=python2.7
set -- "$@" --python-bin python2.7 \
--python-easyinstall "$EASY_INSTALL2" \
--python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
# necessary arguments to fpm's command line later, after we're
# done handling positional arguments.
PACKAGE_TYPE=python
+ python=python3
set -- "$@" --python-bin python3 \
--python-easyinstall "$EASY_INSTALL3" \
--python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
# packages cleanup on upgrade depends on files being listed on the %files
# section in the generated SPEC files. To remove DIRECTORIES, they need to
# be listed in that sectiontoo, so we need to add this parameter to properly
- # remove lingering dirs.
- if [[ rpm = "$FORMAT" ]]; then
+ # remove lingering dirs. But this only works for python2: if used on
+ # python33, it includes dirs like /opt/rh/python33 that belong to
+ # other packages.
+ if [[ "$FORMAT" = rpm ]] && [[ "$python" = python2.7 ]]; then
COMMAND_ARR+=('--rpm-auto-add-directories')
fi
export GOPATH
mkdir -p "$GOPATH/src/git.curoverse.com"
rmdir --parents "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
-ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+ln -snfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
|| fatal "symlink failed"
go get -v github.com/kardianos/govendor \
|| fatal "govendor install failed"
hints:
DockerRequirement:
- dockerPull: biodckr/bwa
+ dockerPull: lh3lh3/bwa
-baseCommand: [bwa, mem]
+baseCommand: [mem]
arguments:
- {prefix: "-t", valueFrom: $(runtime.cores)}
if !ok {
return os.ErrNotExist
}
+ if locked[oldinode] {
+ // oldinode cannot become a descendant of itself.
+ return ErrInvalidArgument
+ }
if existing, ok := newdn.inodes[newname]; ok {
// overwriting an existing file or dir
if dn, ok := existing.(*dirnode); ok {
c.Check(err, check.IsNil)
}
+func (s *CollectionFSSuite) TestRenameError(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("first", 0755)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("first/second", 0755)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("first/second/file", os.O_CREATE|os.O_WRONLY, 0755)
+ c.Assert(err, check.IsNil)
+ f.Write([]byte{1, 2, 3, 4, 5})
+ f.Close()
+ err = fs.Rename("first", "first/second/third")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = fs.Rename("first", "first/third")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = fs.Rename("first/second", "second")
+ c.Check(err, check.IsNil)
+ f, err = fs.OpenFile("second/file", 0, 0)
+ c.Assert(err, check.IsNil)
+ data, err := ioutil.ReadAll(f)
+ c.Check(err, check.IsNil)
+ c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
+}
+
func (s *CollectionFSSuite) TestRename(c *check.C) {
fs, err := (&Collection{}).FileSystem(s.client, s.kc)
c.Assert(err, check.IsNil)
// purposes.
func (runner *ContainerRunner) LogNodeInfo() (err error) {
w := runner.NewLogWriter("node-info")
- logger := log.New(w, "node-info", 0)
commands := []infoCommand{
{
}
// Run commands with informational output to be logged.
- var out []byte
for _, command := range commands {
- out, err = exec.Command(command.cmd[0], command.cmd[1:]...).CombinedOutput()
- if err != nil {
- return fmt.Errorf("While running command %q: %v",
- command.cmd, err)
- }
- logger.Println(command.label)
- for _, line := range strings.Split(string(out), "\n") {
- logger.Println(" ", line)
+ fmt.Fprintln(w, command.label)
+ cmd := exec.Command(command.cmd[0], command.cmd[1:]...)
+ cmd.Stdout = w
+ cmd.Stderr = w
+ if err := cmd.Run(); err != nil {
+ err = fmt.Errorf("While running command %q: %v", command.cmd, err)
+ fmt.Fprintln(w, err)
+ return err
}
+ fmt.Fprintln(w, "")
}
err = w.Close()
c.Check(api.CalledWith("container.log", nil), NotNil)
c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
- c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "foo\n"), Equals, true)
+ c.Check(api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
}
func (s *TestSuite) TestFullRunSetEnv(c *C) {
closing bool
}
-func (arvlog *ArvLogWriter) Write(p []byte) (n int, err error) {
+func (arvlog *ArvLogWriter) Write(p []byte) (int, error) {
// Write to the next writer in the chain (a file in Keep)
var err1 error
if arvlog.writeCloser != nil {
// write to API after checking rate limit
now := time.Now()
- bytesWritten := 0
if now.After(arvlog.logThrottleResetTime) {
// It has been more than throttle_period seconds since the last
"properties": map[string]string{"text": arvlog.bufToFlush.String()}}}
err2 := arvlog.ArvClient.Create("logs", lr, nil)
- bytesWritten = arvlog.bufToFlush.Len()
arvlog.bufToFlush = bytes.Buffer{}
arvlog.bufFlushedAt = now
}
}
- return bytesWritten, nil
+ return len(p), nil
}
// Close the underlying writer
if obj.in_use():
_logger.debug("InodeCache cannot clear inode %i, in use", obj.inode)
return
- obj.kernel_invalidate()
if obj.has_ref(True):
- _logger.debug("InodeCache sent kernel invalidate inode %i", obj.inode)
+ _logger.debug("InodeCache cannot clear inode %i, still referenced", obj.inode)
return
+ obj.kernel_invalidate()
+ _logger.debug("InodeCache sent kernel invalidate inode %i", obj.inode)
obj.clear()
# The llfuse lock is released in del_entry(), which is called by
cache.touch(ent3)
self.assertFalse(ent1.clear.called)
self.assertFalse(ent3.clear.called)
- self.assertTrue(ent3.kernel_invalidate.called)
+ self.assertFalse(ent3.kernel_invalidate.called)
self.assertEqual(1100, cache.total())
# ent1 still in use, ent3 doesn't have ref,
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+[Unit]
+Description=Arvados healthcheck server
+Documentation=https://doc.arvados.org/
+After=network.target
+AssertPathExists=/etc/arvados/config.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/arvados-health
+Restart=always
+RestartSec=1s
+
+[Install]
+WantedBy=multi-user.target
cmd: "move newdir0/testfile newdir1/\n",
match: `(?ms).*Moving .* succeeded.*`,
},
+ {
+ path: writePath,
+ cmd: "move newdir1 newdir1/\n",
+ match: `(?ms).*Moving .* failed.*`,
+ },
+ {
+ path: writePath,
+ cmd: "get newdir1/testfile '" + checkfile.Name() + "'\n",
+ match: `(?ms).*succeeded.*`,
+ data: testdata,
+ },
{
path: writePath,
cmd: "put '" + localfile.Name() + "' newdir1/testfile1\n",
projects / arvados.git / commitdiff