--- /dev/null
+test-package-python27-python-arvados-cwl-runner.sh
\ No newline at end of file
--- /dev/null
+test-package-python27-python-arvados-fuse.sh
\ No newline at end of file
--- /dev/null
+test-package-python27-python-arvados-python-client.sh
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+
+exec python <<EOF
+import arvados_cwl
+print "arvados-cwl-runner version", arvados_cwl.__version__
+EOF
#!/bin/sh
-exec python <<EOF
+exec python2.7 <<EOF
import arvados
print "Successfully imported arvados"
EOF
# clean up the docker build environment
cd "$WORKSPACE"
-tools/arvbox/bin/arvbox rebuild localdemo
+tools/arvbox/bin/arvbox build localdemo
ECODE=$?
if [[ "$ECODE" != "0" ]]; then
echo >&2 "usage: $0 [options]"
echo >&2
echo >&2 "$0 options:"
- echo >&2 " -t, --tags [csv_tags] comma separated tags"
echo >&2 " -u, --upload Upload the images (docker push)"
+ echo >&2 " --no-cache Don't use build cache"
echo >&2 " -h, --help Display this help and exit"
echo >&2
echo >&2 " If no options are given, just builds the images."
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
TEMP=`getopt -o hut: \
- --long help,upload,tags: \
+ --long help,upload,no-cache,tags: \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
upload=true
shift
;;
+ --no-cache)
+ NOCACHE=--no-cache
+ shift
+ ;;
-t | --tags)
case "$2" in
"")
exit 1
;;
*)
- tags=$2;
+ echo "WARNING: --tags is deprecated and doesn't do anything";
shift 2
;;
esac
. $WORKSPACE/build/run-library.sh
docker_push () {
- if [[ ! -z "$tags" ]]
- then
- for tag in $( echo $tags|tr "," " " )
- do
- $DOCKER tag $1 $1:$tag
- done
- fi
-
# Sometimes docker push fails; retry it a few times if necessary.
for i in `seq 1 5`; do
$DOCKER push $*
# clean up the docker build environment
cd "$WORKSPACE"
-cd docker/jobs
-if [[ ! -z "$tags" ]]; then
- docker build --build-arg COMMIT=${tags/,*/} -t arvados/jobs .
+
+python_sdk_ts=$(cd sdk/python && timestamp_from_git)
+cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+
+python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)-2
+cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)-3
+
+if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+ cwl_runner_version=$python_sdk_version
+ gittag=$(cd sdk/python && git log --first-parent --max-count=1 --format=format:%H)
else
- docker build -t arvados/jobs .
+ gittag=$(cd sdk/cwl && git log --first-parent --max-count=1 --format=format:%H)
fi
+echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+
+cd docker/jobs
+docker build $NOCACHE \
+ --build-arg python_sdk_version=$python_sdk_version \
+ --build-arg cwl_runner_version=$cwl_runner_version \
+ -t arvados/jobs:$gittag .
+
ECODE=$?
if [[ "$ECODE" != "0" ]]; then
checkexit $ECODE "docker build"
title "docker build complete (`timer`)"
+if [[ "$ECODE" != "0" ]]; then
+ exit_cleanly
+fi
+
+timer_reset
+
+if docker --version |grep " 1\.[0-9]\." ; then
+ # Docker version prior 1.10 require -f flag
+ # -f flag removed in Docker 1.12
+ FORCE=-f
+fi
+
+docker tag $FORCE arvados/jobs:$gittag arvados/jobs:latest
+
+ECODE=$?
+
+if [[ "$ECODE" != "0" ]]; then
+ EXITCODE=$(($EXITCODE + $ECODE))
+fi
+
+checkexit $ECODE "docker tag"
+title "docker tag complete (`timer`)"
+
title "uploading images"
timer_reset
if [[ "$ECODE" != "0" ]]; then
- title "upload arvados images SKIPPED because build failed"
+ title "upload arvados images SKIPPED because build or tag failed"
else
if [[ $upload == true ]]; then
## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
- docker_push arvados/jobs
+ docker_push arvados/jobs:$gittag
+ docker_push arvados/jobs:latest
title "upload arvados images finished (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set (`timer`)"
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
- cachecontrol)
+ cachecontrol requests)
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
;;
debian8)
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro isodate ruamel.ordereddict
- cachecontrol)
+ cachecontrol requests)
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
;;
ubuntu1404)
# cwl-runner
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/cwl/build"
-fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 3
+fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration 3
fpm_build lockfile "" "" python 0.12.2 --epoch 1
version_from_git() {
# Generates a version number from the git log for the current working
# directory, and writes it to stdout.
- local git_ts git_hash
+ local git_ts git_hash prefix
+ if [[ -n "$1" ]] ; then
+ prefix="$1"
+ else
+ prefix="0.1"
+ fi
+
declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "0.1.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+ echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
}
nohash_version_from_git() {
- version_from_git | cut -d. -f1-3
+ version_from_git $1 | cut -d. -f1-3
}
timestamp_from_git() {
RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3
-ARG COMMIT=latest
-RUN echo $COMMIT && apt-get update -q
+ARG python_sdk_version
+ARG cwl_runner_version
+RUN echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
-RUN apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs python-arvados-cwl-runner
+RUN apt-get update -q
+RUN apt-get install -yq --no-install-recommends \
+ git python-pip python-virtualenv \
+ python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs \
+ python-arvados-python-client=$python_sdk_version \
+ python-arvados-cwl-runner=$cwl_runner_version
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
}
}
}
- if (defined($filename) and ($filename =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
+ if (defined($filename) and ($filename =~ /^((?:sha256:)?[0-9A-Fa-f]{64})\.tar$/)) {
return ($streamname, $1);
} else {
return (undef, undef);
from .fsaccess import CollectionFsAccess
from .perf import Perf
from .pathmapper import FinalOutputPathMapper
+from ._version import __version__
from cwltool.pack import pack
from cwltool.process import shortname, UnsupportedRequirement, getListing
arvpkg = pkg_resources.require("arvados-python-client")
cwlpkg = pkg_resources.require("cwltool")
- return "%s %s, %s %s, %s %s" % (sys.argv[0], arvcwlpkg[0].version,
+ return "%s %s %s, %s %s, %s %s" % (sys.argv[0], __version__, arvcwlpkg[0].version,
"arvados-python-client", arvpkg[0].version,
"cwltool", cwlpkg[0].version)
from .arvdocker import arv_docker_get_image
from . import done
-from .runner import Runner
+from .runner import Runner, arvados_jobs_image
logger = logging.getLogger('arvados.cwl-runner')
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if not docker_req:
- docker_req = {"dockerImageId": "arvados/jobs"}
+ docker_req = {"dockerImageId": arvados_jobs_image(self.arvrunner)}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
workflowcollection = workflowcollection[5:workflowcollection.index('/')]
jobpath = "/var/lib/cwl/job/cwl.input.json"
- container_image = arv_docker_get_image(self.arvrunner.api,
- {"dockerImageId": "arvados/jobs"},
- pull_image,
- self.arvrunner.project_uuid)
-
command = ["arvados-cwl-runner", "--local", "--api=containers"]
if self.output_name:
command.append("--output-name=" + self.output_name)
"cwd": "/var/spool/cwl",
"priority": 1,
"state": "Committed",
- "container_image": container_image,
+ "container_image": arvados_jobs_image(self.arvrunner),
"mounts": {
"/var/lib/cwl/workflow": {
"kind": "collection",
image_tag=image_tag)
if not images:
- imageId = cwltool.docker.get_image(dockerRequirement, pull_image)
+ # Fetch Docker image if necessary.
+ cwltool.docker.get_image(dockerRequirement, pull_image)
+
+ # Upload image to Arvados
args = ["--project-uuid="+project_uuid, image_name]
if image_tag:
args.append(image_tag)
except SystemExit:
raise WorkflowException()
- images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
- image_name=image_name,
- image_tag=image_tag)
+ images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
+ image_name=image_name,
+ image_tag=image_tag)
- #return dockerRequirement["dockerImageId"]
+ if not images:
+ raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
return pdh
import arvados.collection
from .arvdocker import arv_docker_get_image
-from .runner import Runner
+from .runner import Runner, arvados_jobs_image
from .pathmapper import InitialWorkDirPathMapper
from .perf import Perf
from . import done
+from ._version import __version__
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
if docker_req and kwargs.get("use_container") is not False:
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
else:
- runtime_constraints["docker_image"] = "arvados/jobs"
+ runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
resources = self.builder.resources
if resources is not None:
self.job_order["arv:output_name"] = self.output_name
return {
"script": "cwl-runner",
- "script_version": "master",
+ "script_version": __version__,
"repository": "arvados",
"script_parameters": self.job_order,
"runtime_constraints": {
- "docker_image": "arvados/jobs"
+ "docker_image": arvados_jobs_image(self.arvrunner)
}
}
def run(self, *args, **kwargs):
job_spec = self.arvados_job_spec(*args, **kwargs)
+ job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
+
+ job = self.arvrunner.api.jobs().create(
+ body=job_spec,
+ find_or_create=self.enable_reuse
+ ).execute(num_retries=self.arvrunner.num_retries)
+
for k,v in job_spec["script_parameters"].items():
if v is False or v is None or isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
+ del job_spec["owner_uuid"]
+ job_spec["job"] = job
self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
body={
"owner_uuid": self.arvrunner.project_uuid,
self.uuid = self.arvrunner.pipeline["uuid"]
return
- job = None
- while not job:
- time.sleep(2)
- self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().get(
- uuid=self.arvrunner.pipeline["uuid"]).execute(
- num_retries=self.arvrunner.num_retries)
- job = self.arvrunner.pipeline["components"]["cwl-runner"].get("job")
- if not job and self.arvrunner.pipeline["state"] != "RunningOnServer":
- raise WorkflowException("Submitted pipeline is %s" % (self.arvrunner.pipeline["state"]))
-
self.uuid = job["uuid"]
self.arvrunner.processes[self.uuid] = self
from .arvdocker import arv_docker_get_image
from .pathmapper import ArvPathMapper
+from ._version import __version__
logger = logging.getLogger('arvados.cwl-runner')
return workflowmapper
+def arvados_jobs_image(arvrunner):
+ img = "arvados/jobs:"+__version__
+ try:
+ arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ except Exception as e:
+ raise Exception("Docker image %s is not available\n%s" % (img, e) )
+ return img
class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse, output_name):
+++ /dev/null
-../python/gittaggers.py
\ No newline at end of file
--- /dev/null
+from setuptools.command.egg_info import egg_info
+import subprocess
+import time
+import os
+
+SETUP_DIR = os.path.dirname(__file__) or '.'
+
+def choose_version_from():
+ sdk_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
+ cwl_ts = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', SETUP_DIR]).strip()
+ if int(sdk_ts) > int(cwl_ts):
+ getver = os.path.join(SETUP_DIR, "../python")
+ else:
+ getver = SETUP_DIR
+ return getver
+
+class EggInfoFromGit(egg_info):
+ """Tag the build with git commit timestamp.
+
+ If a build tag has already been set (e.g., "egg_info -b", building
+ from source package), leave it alone.
+ """
+
+ def git_timestamp_tag(self):
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', choose_version_from()]).strip()
+ return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
+
+ def tags(self):
+ if self.tag_build is None:
+ self.tag_build = self.git_timestamp_tag()
+ return egg_info.tags(self)
import os
import sys
+import subprocess
import setuptools.command.egg_info as egg_info_cmd
from setuptools import setup, find_packages
except ImportError:
tagger = egg_info_cmd.egg_info
+versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py")
+try:
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%H', gittaggers.choose_version_from()]).strip()
+ with open(versionfile, "w") as f:
+ f.write("__version__ = '%s'\n" % gitinfo)
+except Exception as e:
+ # When installing from package, it won't be part of a git repository, and
+ # check_output() will raise an exception. But the package should include the
+ # version file, so we can proceed.
+ if not os.path.exists(versionfile):
+ raise
+
setup(name='arvados-cwl-runner',
version='1.0',
description='Arvados Common Workflow Language runner',
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
- def test_run(self):
+ @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+ def test_run(self, list_images_in_arv):
runner = mock.MagicMock()
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+ runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
tool = {
"inputs": [],
"outputs": [],
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
'min_cores_per_node': 1,
'min_ram_mb_per_node': 1024,
'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']]
+ ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]]
)
# The test passes some fields in builder.resources
# For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
- def test_resource_requirements(self):
+ @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+ def test_resource_requirements(self, list_images_in_arv):
runner = mock.MagicMock()
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
arvados_cwl.add_arv_hints()
+ list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+ runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
'repository': 'arvados',
'script': 'crunchrunner',
'runtime_constraints': {
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
'min_cores_per_node': 3,
'min_ram_mb_per_node': 3000,
'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']])
+ ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]])
@mock.patch("arvados.collection.CollectionReader")
def test_done(self, reader):
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.collection.Collection")
- def test_run(self, mockcollection):
+ @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
+ def test_run(self, list_images_in_arv, mockcollection):
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
runner = arvados_cwl.ArvCwlRunner(api)
self.assertEqual(runner.work_api, 'jobs')
+ list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
+ runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
with open("tests/wf/scatter2_subwf.cwl") as f:
subwf = f.read()
- mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
- mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
-
runner.api.jobs().create.assert_called_with(
body={
'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
'runtime_constraints': {
'min_scratch_mb_per_node': 2048,
'min_cores_per_node': 1,
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
'min_ram_mb_per_node': 1024
},
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
- ['docker_image_locator', 'in docker', 'arvados/jobs']],
+ ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
find_or_create=True)
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
+
def test_default_work_api(self):
arvados_cwl.add_arv_hints()
}
stubs.expect_job_spec = {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs'
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
},
'script_parameters': {
'x': {
'99999999999999999999999999999991+99/wf/submit_wf.cwl'
},
'repository': 'arvados',
- 'script_version': 'master',
+ 'script_version': arvados_cwl.__version__,
'script': 'cwl-runner'
}
stubs.pipeline_component = stubs.expect_job_spec.copy()
'state': 'RunningOnServer',
"components": {
"cwl-runner": {
- 'runtime_constraints': {'docker_image': 'arvados/jobs'},
+ 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
'script_parameters': {
'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
},
'repository': 'arvados',
- 'script_version': 'master',
- 'script': 'cwl-runner'
+ 'script_version': arvados_cwl.__version__,
+ 'script': 'cwl-runner',
+ 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
}
}
}
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
'name': 'submit_wf.cwl',
- 'container_image': '99999999999999999999999999999993+99',
+ 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
'output_path': '/var/spool/cwl',
'cwd': '/var/spool/cwl',
'runtime_constraints': {
"components": {
"inputs_test.cwl": {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs',
+ 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
},
'script_parameters': {
'cwl:tool':
},
},
'repository': 'arvados',
- 'script_version': 'master',
+ 'script_version': arvados_cwl.__version__,
'script': 'cwl-runner',
},
},
start_banner = "### BEGIN Arvados-managed keys -- changes between markers will be overwritten\n"
end_banner = "### END Arvados-managed keys -- changes between markers will be overwritten\n"
-keys = ''
+# Don't try to create any local accounts
+skip_missing_users = ARGV.index("--skip-missing-users")
-seen = Hash.new
+keys = ''
begin
- uids = Hash[Etc.to_enum(:passwd).map { |ent| [ent.name, ent.uid] }]
- gids = Hash[Etc.to_enum(:group).map { |ent| [ent.name, ent.gid] }]
arv = Arvados.new({ :suppress_ssl_warnings => false })
vm_uuid = ENV['ARVADOS_VIRTUAL_MACHINE_UUID']
uid_min = new_uid_min if (new_uid_min > 0)
end
end
- logins.reject! { |l| (uids[l[:username]] || 65535) < uid_min }
+ pwnam = Hash.new()
+ logins.reject! do |l|
+ return false if pwnam[l[:username]]
+ begin
+ pwnam[l[:username]] = Etc.getpwnam(l[:username])
+ rescue
+ if skip_missing_users
+ STDERR.puts "Account #{l[:username]} not found. Skipping"
+ true
+ end
+ else
+ if pwnam[l[:username]].uid < uid_min
+ STDERR.puts "Account #{l[:username]} uid #{pwnam[l[:username]].uid} < uid_min #{uid_min}. Skipping"
+ true
+ end
+ end
+ end
keys = Hash.new()
# Collect all keys
logins.each do |l|
next if seen[l[:username]]
- seen[l[:username]] = true if not seen.has_key?(l[:username])
+ seen[l[:username]] = true
- unless uids[l[:username]]
+ unless pwnam[l[:username]]
STDERR.puts "Creating account #{l[:username]}"
groups = l[:groups] || []
# Adding users to the FUSE group has long been hardcoded behavior.
groups << "fuse"
- groups.select! { |name| gids[name] }
+ groups.select! { |g| Etc.getgrnam(g) rescue false }
# Create new user
- next unless system("useradd", "-m",
- "-c", l[:username],
- "-s", "/bin/bash",
- "-G", groups.join(","),
- l[:username],
- out: devnull)
+ unless system("useradd", "-m",
+ "-c", l[:username],
+ "-s", "/bin/bash",
+ "-G", groups.join(","),
+ l[:username],
+ out: devnull)
+ STDERR.puts "Account creation failed for #{l[:username]}: $?"
+ next
+ end
+ begin
+ pwnam[l[:username]] = Etc.getpwnam(l[:username])
+ rescue => e
+ STDERR.puts "Created account but then getpwnam() failed for #{l[:username]}: #{e}"
+ raise
+ end
end
- # Create .ssh directory if necessary
- @homedir = Etc.getpwnam(l[:username]).dir
+
+ @homedir = pwnam[l[:username]].dir
userdotssh = File.join(@homedir, ".ssh")
Dir.mkdir(userdotssh) if !File.exists?(userdotssh)
f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,fuse adminroot'
f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,admin,fuse adminroot'
end
- $stderr.puts "*** Expect crash in dir_s_mkdir:"
+ $stderr.puts "*** Expect crash after getpwnam() fails:"
invoke_sync binstubs: ['new_user']
assert !$?.success?
spied = File.read(@tmpdir+'/spy')
CONFIG=$1
TAG=$2
+ shift
+
if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then
echo "Container $ARVBOX_CONTAINER is already running"
exit 0
echo "Container $ARVBOX_CONTAINER already exists but is not running; use restart or rebuild"
exit 1
fi
-
+
if test ! -z "$TAG"
then
- TAG=":$TAG"
+ if test $(echo $TAG | cut -c1-1) != '-' ; then
+ TAG=":$TAG"
+ shift
+ else
+ unset TAG
+ fi
fi
if echo "$CONFIG" | grep '^public' ; then
echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)"
exit 1
fi
- docker build $NO_CACHE -t arvados/arvbox-base -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
+ GITHEAD=$(cd $ARVBOX_DOCKER && git log --format=%H -n1 HEAD)
+ docker build --build-arg=arvados_version=$GITHEAD $NO_CACHE -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER"
+ if docker --version |grep " 1\.[0-9]\." ; then
+ # Docker version prior 1.10 require -f flag
+ # -f flag removed in Docker 1.12
+ FORCE=-f
+ fi
if test "$1" = localdemo -o "$1" = publicdemo ; then
- docker build $NO_CACHE -t arvados/arvbox-demo -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
+ docker build $NO_CACHE -t arvados/arvbox-demo:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER"
+ docker tag $FORCE arvados/arvbox-demo:$GITHEAD arvados/arvbox-demo:latest
else
- docker build $NO_CACHE -t arvados/arvbox-dev -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
+ docker build $NO_CACHE -t arvados/arvbox-dev:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER"
+ docker tag $FORCE arvados/arvbox-dev:$GITHEAD arvados/arvbox-dev:latest
fi
}
cat)
if test -n "$1" ; then
- exec docker exec -ti $ARVBOX_CONTAINER cat "$@"
+ exec docker exec $ARVBOX_CONTAINER cat "$@"
else
echo "Usage: $0 $subcmd <files>"
fi
libjson-perl nginx gitolite3 lsof \
apt-transport-https ca-certificates slurm-wlm
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
+ linkchecker python3-virtualenv python-virtualenv xvfb iceweasel
+
RUN cd /usr/local && \
- curl -O http://storage.googleapis.com/golang/go1.6.2.linux-amd64.tar.gz && \
- tar -xzf go1.6.2.linux-amd64.tar.gz && \
- rm go1.6.2.linux-amd64.tar.gz && \
+ GOVERSION=1.7.1 && \
+ curl -O http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz && \
+ tar -xzf go${GOVERSION}.linux-amd64.tar.gz && \
+ rm go${GOVERSION}.linux-amd64.tar.gz && \
cd bin && \
ln -s /usr/local/go/bin/* .
GOPATH=$PWD go get github.com/curoverse/runsvinit && \
install bin/runsvinit /usr/local/bin
+RUN set -e && \
+ PJS=phantomjs-1.9.7-linux-x86_64 && \
+ curl -L -o/tmp/$PJS.tar.bz2 http://cache.arvados.org/$PJS.tar.bz2 && \
+ tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
+ ln -s ../$PJS/bin/phantomjs /usr/local/bin/
+
+ARG arvados_version
+RUN echo arvados_version is git commit $arvados_version
+
ADD fuse.conf /etc/
ADD crunch-setup.sh gitolite.rc \
FROM arvados/arvbox-base
-ARG arvados_version=master
+ARG arvados_version
ARG sso_version=master
RUN cd /usr/src && \
FROM arvados/arvbox-base
-
-RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends install \
- linkchecker python3-virtualenv python-virtualenv xvfb iceweasel
-
-RUN set -e && \
- PJS=phantomjs-1.9.7-linux-x86_64 && \
- curl -L -o/tmp/$PJS.tar.bz2 http://cache.arvados.org/$PJS.tar.bz2 && \
- tar -C /usr/local -xjf /tmp/$PJS.tar.bz2 && \
- ln -s ../$PJS/bin/phantomjs /usr/local/bin/
+ARG arvados_version
ADD service/ /var/lib/arvbox/service
RUN rmdir /etc/service && ln -sf /var/lib/arvbox/service /etc
pip_install() {
pushd /var/lib/pip
- for p in $(ls http*.tar.gz) ; do
- if test -f $p ; then
- ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
- fi
- done
- for p in $(ls http*.whl) ; do
+ for p in $(ls http*.tar.gz) $(ls http*.whl) $(ls http*.zip) ; do
if test -f $p ; then
ln -sf $p $(echo $p | sed 's/.*%2F\(.*\)/\1/')
fi
python setup.py sdist
pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1)
-cd /usr/src/arvados/sdk/cwl
-python setup.py sdist
-pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
-
cd /usr/src/arvados/services/fuse
python setup.py sdist
pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1)
+
+cd /usr/src/arvados/sdk/cwl
+python setup.py sdist
+pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1)
projects / arvados.git / commitdiff