.DS_Store
.vscode
.Rproj.user
+_version.py
\ No newline at end of file
timer_reset
if [ ${#failures[@]} -eq 0 ]; then
+ echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
else
echo "Skipping package upload, there were errors building and/or testing the packages"
fi
declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
-}
+ ARVADOS_BUILDING_VERSION="$(git describe --abbrev=0).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)"
+ echo "$ARVADOS_BUILDING_VERSION"
+}
nohash_version_from_git() {
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_VERSION"
return
fi
- version_from_git $1 | cut -d. -f1-3
+ version_from_git $1 | cut -d. -f1-4
}
timestamp_from_git() {
# See if we can skip building the package, only if it already exists in the
# processed/ directory. If so, move it back to the packages directory to make
# sure it gets picked up by the test and/or upload steps.
- if [[ -e "processed/$complete_pkgname" ]]; then
- echo "Package $complete_pkgname exists, not rebuilding!"
- mv processed/$complete_pkgname .
- return 1
+ # Get the list of packages from the repos
+
+ if [[ "$FORMAT" == "deb" ]]; then
+ debian_distros="jessie precise stretch trusty wheezy xenial"
+
+ for D in ${debian_distros}; do
+ if [ ${pkgname:0:3} = "lib" ]; then
+ repo_subdir=${pkgname:0:4}
+ else
+ repo_subdir=${pkgname:0:1}
+ fi
+
+ repo_pkg_list=$(curl -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/)
+ echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+ if [ $? -eq 0 ]; then
+ echo "Package $complete_pkgname exists, not rebuilding!"
+ curl -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname}
+ return 1
+ else
+ echo "Package $complete_pkgname not found, building"
+ return 0
+ fi
+ done
else
- echo "Package $complete_pkgname not found, building"
- return 0
+ centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/"
+
+ repo_pkg_list=$(curl -o - ${centos_repo})
+ echo ${repo_pkg_list} |grep -q ${complete_pkgname}
+ if [ $? -eq 0 ]; then
+ echo "Package $complete_pkgname exists, not rebuilding!"
+ curl -o ./${complete_pkgname} ${centos_repo}${complete_pkgname}
+ return 1
+ else
+ echo "Package $complete_pkgname not found, building"
+ return 0
+ fi
fi
}
# If the authorization origins are not displayed, clicking on *Create Client ID* will take you to *Consent screen* settings.
## On consent screen settings, enter the appropriate details and click on *Save*.
## This will return you to the *Create Client ID* dialog box.
-# You must set the authorization origins. Edit @sso.your-site.com@ to the appropriate hostname that you will use to access the SSO service:
-## JavaScript origin should be @https://sso.your-site.com/@
-## Redirect URI should be @https://sso.your-site.com/users/auth/google_oauth2/callback@
+# You must set the authorization origins. Edit @auth.your.domain@ to the appropriate hostname that you will use to access the SSO service:
+## JavaScript origin should be @https://auth.your.domain/@
+## Redirect URI should be @https://auth.your.domain/users/auth/google_oauth2/callback@
# Copy the values of *Client ID* and *Client secret* from the Google Developers Console into the Google section of @config/application.yml@, like this:
<notextile>
include LICENSE-2.0.txt
include README.rst
+include arvados_version.py
\ No newline at end of file
# Reload tool object which may have been updated by
# upload_workflow_deps
+ # Don't validate this time because it will just print redundant errors.
tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
makeTool=self.arv_make_tool,
loader=tool.doc_loader,
avsc_names=tool.doc_schema,
- metadata=tool.metadata)
+ metadata=tool.metadata,
+ do_validate=False)
# Upload local file references in the job order.
job_order = upload_job_order(self, "%s input" % kwargs["name"],
help="Workflow priority (range 1..1000, higher has precedence over lower, containers api only)",
default=DEFAULT_PRIORITY)
+ parser.add_argument("--disable-validate", dest="do_validate",
+ action="store_false", default=True,
+ help=argparse.SUPPRESS)
+
+ parser.add_argument("--disable-js-validation",
+ action="store_true", default=False,
+ help=argparse.SUPPRESS)
+
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--trash-intermediate", action="store_true",
default=False, dest="trash_intermediate",
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import pkg_resources
-
-__version__ = pkg_resources.require('arvados-cwl-runner')[0].version
container_req["properties"]["template_uuid"] = self.tool.tool["id"][6:33]
- command = ["arvados-cwl-runner", "--local", "--api=containers", "--no-log-timestamps"]
+ # --local means execute the workflow instead of submitting a container request
+ # --api=containers means use the containers API
+ # --no-log-timestamps means don't add timestamps (the logging infrastructure does this)
+ # --disable-validate because we already validated so don't need to do it again
+ command = ["arvados-cwl-runner", "--local", "--api=containers", "--no-log-timestamps", "--disable-validate"]
if self.output_name:
command.append("--output-name=" + self.output_name)
container_req["output_name"] = self.output_name
if isinstance(res_req[a], int): # integer check
all_res_req[a].append(res_req[a])
else:
- msg = SourceLine(res_req).makeError(
+ msg = SourceLine(res_req, a).makeError(
"Non-top-level ResourceRequirement in single container cannot have expressions")
exception_msgs.append(msg)
if exception_msgs:
self.arvrunner = arvrunner
self.work_api = kwargs["work_api"]
self.wf_pdh = None
+ self.dynamic_resource_req = []
+ self.static_resource_req = []
def job(self, joborder, output_callback, **kwargs):
kwargs["work_api"] = self.work_api
builder.hints = workflowobj["hints"]
builder.resources = {}
- res_reqs = {"requirements": [], "hints": []}
- for t in ("requirements", "hints"):
- for item in packed["$graph"]:
- if t in item:
- if item["id"] == "#main": # evaluate potential expressions in the top-level requirements/hints
- for req in item[t]:
- if req["class"] == "ResourceRequirement":
- eval_req = {"class": "ResourceRequirement"}
- for a in max_res_pars + sum_res_pars:
- if a in req:
- eval_req[a] = builder.do_eval(req[a])
- res_reqs[t].append(eval_req)
- else:
- for req in item[t]:
- if req["class"] == "ResourceRequirement":
- res_reqs[t].append(req)
- overall_res_req = {"requirements": get_overall_res_req(res_reqs["requirements"]),
- "hints": get_overall_res_req(res_reqs["hints"])}
-
- new_spec = {"requirements": self.requirements, "hints": self.hints}
- for t in ("requirements", "hints"):
- for req in new_spec[t]:
- if req["class"] == "ResourceRequirement":
- new_spec[t].remove(req)
- if overall_res_req[t]:
- new_spec[t].append(overall_res_req[t])
+ def visit(item):
+ for t in ("hints", "requirements"):
+ if t not in item:
+ continue
+ for req in item[t]:
+ if req["class"] == "ResourceRequirement":
+ dyn = False
+ for k in max_res_pars + sum_res_pars:
+ if k in req:
+ if isinstance(req[k], basestring):
+ if item["id"] == "#main":
+ # only the top-level requirements/hints may contain expressions
+ self.dynamic_resource_req.append(req)
+ dyn = True
+ break
+ else:
+ with SourceLine(req, k, WorkflowException):
+ raise WorkflowException("Non-top-level ResourceRequirement in single container cannot have expressions")
+ if not dyn:
+ self.static_resource_req.append(req)
+
+ visit_class(packed["$graph"], ("Workflow", "CommandLineTool"), visit)
+
+ if self.static_resource_req:
+ self.static_resource_req = [get_overall_res_req(self.static_resource_req)]
upload_dependencies(self.arvrunner,
kwargs.get("name", ""),
uri,
False)
+ if self.dynamic_resource_req:
+ builder = Builder()
+ builder.job = joborder
+ builder.requirements = self.requirements
+ builder.hints = self.hints
+ builder.resources = {}
+
+ # Evaluate dynamic resource requirements using current builder
+ rs = copy.copy(self.static_resource_req)
+ for dyn_rs in self.dynamic_resource_req:
+ eval_req = {"class": "ResourceRequirement"}
+ for a in max_res_pars + sum_res_pars:
+ if a in dyn_rs:
+ eval_req[a] = builder.do_eval(dyn_rs[a])
+ rs.append(eval_req)
+ job_res_reqs = [get_overall_res_req(rs)]
+ else:
+ job_res_reqs = self.static_resource_req
+
with Perf(metrics, "subworkflow adjust"):
joborder_resolved = copy.deepcopy(joborder)
joborder_keepmount = copy.deepcopy(joborder)
"inputs": self.tool["inputs"],
"outputs": self.tool["outputs"],
"stdout": "cwl.output.json",
- "requirements": self.requirements+[
+ "requirements": self.requirements+job_res_reqs+[
{
"class": "InitialWorkDirRequirement",
"listing": [{
args.trash_intermediate = False
args.intermediate_output_ttl = 0
args.priority = arvados_cwl.DEFAULT_PRIORITY
+ args.do_validate = True
+ args.disable_js_validation = False
runner.arv_executor(t, job_order_object, **vars(args))
except Exception as e:
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+
+def git_latest_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
+def git_timestamp_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+
+def save_version(setup_dir, module, v):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+ except subprocess.CalledProcessError:
+ pass
+
+ return read_version(setup_dir, module)
If a build tag has already been set (e.g., "egg_info -b", building
from source package), leave it alone.
"""
+ def git_latest_tag(self):
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
def git_timestamp_tag(self):
gitinfo = subprocess.check_output(
def tags(self):
if self.tag_build is None:
- self.tag_build = self.git_timestamp_tag()
+ self.tag_build = self.git_latest_tag() + self.git_timestamp_tag()
return egg_info.tags(self)
#
# SPDX-License-Identifier: Apache-2.0
+from __future__ import absolute_import
import os
import sys
-import subprocess
-import setuptools.command.egg_info as egg_info_cmd
from setuptools import setup, find_packages
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-tagger = egg_info_cmd.egg_info
-version = os.environ.get("ARVADOS_BUILDING_VERSION")
-if not version:
- version = "1.0"
- try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
- except ImportError:
- pass
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvados_cwl")
setup(name='arvados-cwl-runner',
version=version,
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180326152342',
+ 'cwltool==1.0.20180403145700',
'schema-salad==2.6.20171201034858',
'typing==3.5.3.0',
'ruamel.yaml==0.13.7',
],
test_suite='tests',
tests_require=['mock>=1.0'],
- zip_safe=True,
- cmdclass={'egg_info': tagger},
+ zip_safe=True
)
}
tool: wf/secret_wf.cwl
doc: "Test secret input parameters"
+
+- job: null
+ output:
+ out: null
+ tool: wf/runin-reqs-wf.cwl
+ doc: "RunInSingleContainer handles dynamic resource requests on step"
+
+- job: null
+ output:
+ out: null
+ tool: wf/runin-reqs-wf2.cwl
+ doc: "RunInSingleContainer handles dynamic resource requests on embedded subworkflow"
+
+- job: null
+ output:
+ out: null
+ tool: wf/runin-reqs-wf3.cwl
+ should_fail: true
+ doc: "RunInSingleContainer disallows dynamic resource request on subworkflow steps"
+
+- job: null
+ output:
+ out: null
+ tool: wf/runin-reqs-wf4.cwl
+ doc: "RunInSingleContainer discovers static resource request in subworkflow steps"
'secret_mounts': {},
'state': 'Committed',
'owner_uuid': None,
- 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ 'command': ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--enable-reuse', '--on-error=continue', '--eval-timeout=20',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'name': 'submit_wf.cwl',
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = [
- 'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ 'arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--disable-reuse', '--on-error=continue', '--eval-timeout=20',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = [
- 'arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ 'arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--disable-reuse', '--on-error=continue', '--eval-timeout=20',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
- '--enable-reuse', '--on-error=stop', '--eval-timeout=20',
- '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ '--enable-reuse', '--on-error=stop', '--eval-timeout=20',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
- "--output-name="+output_name, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
- '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ "--output-name="+output_name, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["output_name"] = output_name
stubs.api.container_requests().create.assert_called_with(
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--enable-reuse', '--on-error=continue',
"--intermediate-output-ttl=3600", '--eval-timeout=20',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--enable-reuse', '--on-error=continue',
"--trash-intermediate", '--eval-timeout=20',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
- "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
- '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue', '--eval-timeout=20',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
'output_path': '/var/spool/cwl',
'name': 'expect_arvworkflow.cwl#main',
'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
- 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ 'command': ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--enable-reuse', '--on-error=continue', '--eval-timeout=20',
'/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
'output_path': '/var/spool/cwl',
'name': 'a test workflow',
'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
- 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ 'command': ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--enable-reuse', '--on-error=continue', '--eval-timeout=20',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["owner_uuid"] = project_uuid
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--enable-reuse', '--on-error=continue', '--project-uuid='+project_uuid, '--eval-timeout=20',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
'--enable-reuse', '--on-error=continue', '--eval-timeout=60.0',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
"--local",
"--api=containers",
"--no-log-timestamps",
+ "--disable-validate",
"--enable-reuse",
"--on-error=continue",
"--eval-timeout=20",
--- /dev/null
+import arvados
+import sys
+import os
+
+if "JOB_UUID" in os.environ:
+ requested = arvados.api().jobs().get(uuid=os.environ["JOB_UUID"]).execute()["runtime_constraints"]["min_ram_mb_per_node"]
+else:
+ requested = arvados.api().containers().current().execute()["runtime_constraints"]["ram"]/(1024*1024)
+
+print("Requested %d expected %d" % (requested, int(sys.argv[1])))
+
+exit(0 if requested == int(sys.argv[1]) else 1)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+inputs:
+ count:
+ type: int[]
+ default: [1, 2, 3, 4]
+ script:
+ type: File
+ default:
+ class: File
+ location: check_mem.py
+outputs:
+ out: []
+requirements:
+ SubworkflowFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ InlineJavascriptRequirement: {}
+ StepInputExpressionRequirement: {}
+steps:
+ substep:
+ in:
+ count: count
+ script: script
+ out: []
+ hints:
+ - class: arv:RunInSingleContainer
+ - class: ResourceRequirement
+ ramMin: $(inputs.count*4)
+ - class: arv:APIRequirement
+ scatter: count
+ run:
+ class: Workflow
+ id: mysub
+ inputs:
+ count: int
+ script: File
+ outputs: []
+ steps:
+ sleep1:
+ in:
+ count: count
+ script: script
+ out: []
+ run:
+ class: CommandLineTool
+ id: subtool
+ inputs:
+ count:
+ type: int
+ script: File
+ outputs: []
+ arguments: [python, $(inputs.script), $(inputs.count * 4)]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+inputs:
+ count:
+ type: int[]
+ default: [1, 2, 3, 4]
+ script:
+ type: File
+ default:
+ class: File
+ location: check_mem.py
+outputs:
+ out: []
+requirements:
+ SubworkflowFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ InlineJavascriptRequirement: {}
+ StepInputExpressionRequirement: {}
+steps:
+ substep:
+ in:
+ count: count
+ script: script
+ out: []
+ hints:
+ - class: arv:RunInSingleContainer
+ - class: arv:APIRequirement
+ scatter: count
+ run:
+ class: Workflow
+ id: mysub
+ inputs:
+ count: int
+ script: File
+ outputs: []
+ hints:
+ - class: ResourceRequirement
+ ramMin: $(inputs.count*4)
+ steps:
+ sleep1:
+ in:
+ count: count
+ script: script
+ out: []
+ run:
+ class: CommandLineTool
+ id: subtool
+ inputs:
+ count:
+ type: int
+ script: File
+ outputs: []
+ arguments: [python, $(inputs.script), $(inputs.count * 4)]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+inputs:
+ count:
+ type: int[]
+ default: [1, 2, 3, 4]
+ script:
+ type: File
+ default:
+ class: File
+ location: check_mem.py
+outputs:
+ out: []
+requirements:
+ SubworkflowFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ InlineJavascriptRequirement: {}
+ StepInputExpressionRequirement: {}
+steps:
+ substep:
+ in:
+ count: count
+ script: script
+ out: []
+ hints:
+ - class: arv:RunInSingleContainer
+ - class: arv:APIRequirement
+ scatter: count
+ run:
+ class: Workflow
+ id: mysub
+ inputs:
+ count: int
+ script: File
+ outputs: []
+ steps:
+ sleep1:
+ in:
+ count: count
+ script: script
+ out: []
+ run:
+ class: CommandLineTool
+ id: subtool
+ hints:
+ - class: ResourceRequirement
+ ramMin: $(inputs.count*4)
+ inputs:
+ count:
+ type: int
+ script: File
+ outputs: []
+ arguments: [python, $(inputs.script), $(inputs.count * 4)]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+inputs:
+ count:
+ type: int[]
+ default: [1, 2, 3, 4]
+ script:
+ type: File
+ default:
+ class: File
+ location: check_mem.py
+outputs:
+ out: []
+requirements:
+ SubworkflowFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ InlineJavascriptRequirement: {}
+ StepInputExpressionRequirement: {}
+steps:
+ substep:
+ in:
+ count: count
+ script: script
+ out: []
+ hints:
+ - class: arv:RunInSingleContainer
+ - class: arv:APIRequirement
+ scatter: count
+ run:
+ class: Workflow
+ id: mysub
+ inputs:
+ count: int
+ script: File
+ outputs: []
+ steps:
+ sleep1:
+ in:
+ count: count
+ script: script
+ out: []
+ run:
+ class: CommandLineTool
+ id: subtool
+ hints:
+ - class: ResourceRequirement
+ ramMin: 8
+ inputs:
+ count:
+ type: int
+ script: File
+ outputs: []
+ arguments: [python, $(inputs.script), "8"]
c.Check(f.Close(), check.IsNil)
m, err := s.fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
c.Check(m, check.Matches, `. 37b51d194a7513e45b56f6524f2d51f2\+3\+\S+ 0:3:new-file\\0401\n./dir1 .* 3:3:bar 0:3:foo\n`)
}
c.Check(err, check.IsNil)
pos, err = f.Seek(0, io.SeekCurrent)
c.Check(pos, check.Equals, int64(18))
+ c.Check(err, check.IsNil)
pos, err = f.Seek(-18, io.SeekCurrent)
+ c.Check(pos, check.Equals, int64(0))
c.Check(err, check.IsNil)
n, err = io.ReadFull(f, buf)
c.Check(n, check.Equals, 18)
// truncate to current size
err = f.Truncate(18)
+ c.Check(err, check.IsNil)
f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
// shrink to block/extent boundary
err = f.Truncate(32)
+ c.Check(err, check.IsNil)
f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
// shrink to partial block/extent
err = f.Truncate(15)
+ c.Check(err, check.IsNil)
f2.Seek(0, io.SeekStart)
buf2, err = ioutil.ReadAll(f2)
c.Check(err, check.IsNil)
checkSize := func(size int64) {
fi, err := f.Stat()
+ c.Assert(err, check.IsNil)
c.Check(fi.Size(), check.Equals, size)
f, err := fs.OpenFile("test", os.O_CREATE|os.O_RDWR, 0755)
c.Assert(err, check.IsNil)
defer f.Close()
fi, err = f.Stat()
+ c.Check(err, check.IsNil)
c.Check(fi.Size(), check.Equals, size)
pos, err := f.Seek(0, io.SeekEnd)
+ c.Check(err, check.IsNil)
c.Check(pos, check.Equals, size)
}
expect := map[string][]byte{
"0": nil,
- "00": []byte{},
- "one": []byte{1},
+ "00": {},
+ "one": {1},
"dir/0": nil,
- "dir/two": []byte{1, 2},
+ "dir/two": {1, 2},
"dir/zero": nil,
"dir/zerodir/zero": nil,
"zero/zero/zero": nil,
c.Assert(err, check.IsNil)
for name, data := range expect {
- f, err := persisted.Open("bogus-" + name)
+ _, err = persisted.Open("bogus-" + name)
c.Check(err, check.NotNil)
- f, err = persisted.Open(name)
+ f, err := persisted.Open(name)
c.Assert(err, check.IsNil)
if data == nil {
c.Check(n, check.Equals, 1)
c.Check(buf[:1], check.DeepEquals, []byte{1})
pos, err = f.Seek(0, io.SeekCurrent)
+ c.Assert(err, check.IsNil)
c.Check(pos, check.Equals, int64(1))
f.Write([]byte{4, 5, 6})
pos, err = f.Seek(0, io.SeekCurrent)
+ c.Assert(err, check.IsNil)
c.Check(pos, check.Equals, int64(6))
f.Seek(0, io.SeekStart)
n, err = f.Read(buf)
c.Check(pos, check.Equals, int64(3))
f.Write([]byte{7, 8, 9})
pos, err = f.Seek(0, io.SeekCurrent)
+ c.Check(err, check.IsNil)
c.Check(pos, check.Equals, int64(9))
f.Close()
for {
if err := os.Chdir("sdk/python/tests"); err == nil {
pythonTestDir, err = os.Getwd()
+ if err != nil {
+ log.Fatal(err)
+ }
return
}
if parent, err := os.Getwd(); err != nil || parent == "/" {
c.Check(offset, check.Equals, int64(a))
buf := make([]byte, b-a)
n, err := io.ReadFull(rdr, buf)
+ c.Check(err, check.IsNil)
c.Check(n, check.Equals, b-a)
c.Check(string(buf), check.Equals, want[a:b])
}
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 1)
}
include examples/shellinabox
include lib/libpam_arvados.py
include pam-configs/arvados
+include arvados_version.py
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+
+def git_latest_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
+def git_timestamp_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+
+def save_version(setup_dir, module, v):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+ except subprocess.CalledProcessError:
+ pass
+
+ return read_version(setup_dir, module)
#
# SPDX-License-Identifier: Apache-2.0
+from __future__ import absolute_import
import glob
import os
import sys
-import setuptools.command.egg_info as egg_info_cmd
+import re
import subprocess
from setuptools import setup, find_packages
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-tagger = egg_info_cmd.egg_info
-version = os.environ.get("ARVADOS_BUILDING_VERSION")
-if not version:
- version = "0.1"
- try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
- except ImportError:
- pass
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvados_pam")
+
+short_tests_only = False
+if '--short-tests-only' in sys.argv:
+ short_tests_only = True
+ sys.argv.remove('--short-tests-only')
setup(name='arvados-pam',
version=version,
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'python-pam'],
- zip_safe=False,
- cmdclass={'egg_info': tagger},
+ zip_safe=False
)
include LICENSE-2.0.txt
include README.rst
+include arvados_version.py
\ No newline at end of file
1. Add this Arvados repository to your sources list::
- deb http://apt.arvados.org/ wheezy main
+ deb http://apt.arvados.org/ stretch main
2. Update your package list.
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import pkg_resources
-
-__version__ = pkg_resources.require('arvados-python-client')[0].version
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+
+def git_latest_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
+def git_timestamp_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+
+def save_version(setup_dir, module, v):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+ except subprocess.CalledProcessError:
+ pass
+
+ return read_version(setup_dir, module)
If a build tag has already been set (e.g., "egg_info -b", building
from source package), leave it alone.
"""
+ def git_latest_tag(self):
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
def git_timestamp_tag(self):
gitinfo = subprocess.check_output(
['git', 'log', '--first-parent', '--max-count=1',
def tags(self):
if self.tag_build is None:
- self.tag_build = self.git_timestamp_tag()
+ self.tag_build = self.git_latest_tag()+self.git_timestamp_tag()
return egg_info.tags(self)
#
# SPDX-License-Identifier: Apache-2.0
+from __future__ import absolute_import
import os
import sys
-import setuptools.command.egg_info as egg_info_cmd
+import re
from setuptools import setup, find_packages
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-tagger = egg_info_cmd.egg_info
-version = os.environ.get("ARVADOS_BUILDING_VERSION")
-if not version:
- version = "0.1"
- try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
- except ImportError:
- pass
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvados")
short_tests_only = False
if '--short-tests-only' in sys.argv:
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
- zip_safe=False,
- cmdclass={'egg_info': tagger},
+ zip_safe=False
)
#
# SPDX-License-Identifier: AGPL-3.0
+require 'arvados_model_updates'
require 'has_uuid'
require 'record_filters'
require 'serializers'
class ArvadosModel < ActiveRecord::Base
self.abstract_class = true
+ include ArvadosModelUpdates
include CurrentApiClient # current_user, current_api_client, etc.
include DbCurrentTime
extend RecordFilters
self.updated_at = current_time
self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
self.modified_at = current_time
- self.modified_by_user_uuid = current_user ? current_user.uuid : nil
+ if !anonymous_updater
+ self.modified_by_user_uuid = current_user ? current_user.uuid : nil
+ end
self.modified_by_client_uuid = current_api_client ? current_api_client.uuid : nil
true
end
# SPDX-License-Identifier: AGPL-3.0
require 'arvados/keep'
-require 'sweep_trashed_collections'
+require 'sweep_trashed_objects'
require 'trashable'
class Collection < ArvadosModel
end
def self.where *args
- SweepTrashedCollections.sweep_if_stale
+ SweepTrashedObjects.sweep_if_stale
super
end
require 'safe_json'
class Container < ArvadosModel
+ include ArvadosModelUpdates
include HasUuid
include KindAndEtag
include CommonApiTemplate
c = Container.create! c_attrs
retryable_requests.each do |cr|
cr.with_lock do
- # Use row locking because this increments container_count
- cr.container_uuid = c.uuid
- cr.save!
+ leave_modified_by_user_alone do
+ # Use row locking because this increments container_count
+ cr.container_uuid = c.uuid
+ cr.save!
+ end
end
end
end
# Notify container requests associated with this container
ContainerRequest.where(container_uuid: uuid,
state: ContainerRequest::Committed).each do |cr|
- cr.finalize!
+ leave_modified_by_user_alone do
+ cr.finalize!
+ end
end
# Cancel outstanding container requests made by this container.
includes(:container).
where(requesting_container_uuid: uuid,
state: ContainerRequest::Committed).each do |cr|
- cr.update_attributes!(priority: 0)
- cr.container.reload
- if cr.container.state == Container::Queued || cr.container.state == Container::Locked
- # If the child container hasn't started yet, finalize the
- # child CR now instead of leaving it "on hold", i.e.,
- # Queued with priority 0. (OTOH, if the child is already
- # running, leave it alone so it can get cancelled the
- # usual way, get a copy of the log collection, etc.)
- cr.update_attributes!(state: ContainerRequest::Final)
+ leave_modified_by_user_alone do
+ cr.update_attributes!(priority: 0)
+ cr.container.reload
+ if cr.container.state == Container::Queued || cr.container.state == Container::Locked
+ # If the child container hasn't started yet, finalize the
+ # child CR now instead of leaving it "on hold", i.e.,
+ # Queued with priority 0. (OTOH, if the child is already
+ # running, leave it alone so it can get cancelled the
+ # usual way, get a copy of the log collection, etc.)
+ cr.update_attributes!(state: ContainerRequest::Final)
+ end
end
end
end
end
end
-
end
require 'whitelist_update'
class ContainerRequest < ArvadosModel
+ include ArvadosModelUpdates
include HasUuid
include KindAndEtag
include CommonApiTemplate
if state == Committed && Container.find_by_uuid(container_uuid).final?
reload
act_as_system_user do
- finalize!
+ leave_modified_by_user_alone do
+ finalize!
+ end
end
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+module ArvadosModelUpdates
+ # ArvadosModel checks this to decide whether it should update the
+ # 'modified_by_user_uuid' field.
+ def anonymous_updater
+ Thread.current[:anonymous_updater] || false
+ end
+
+ def leave_modified_by_user_alone
+ anonymous_updater_was = anonymous_updater
+ begin
+ Thread.current[:anonymous_updater] = true
+ yield
+ ensure
+ Thread.current[:anonymous_updater] = anonymous_updater_was
+ end
+ end
+end
end
def act_as_user user
- #auth_was = Thread.current[:api_client_authorization]
user_was = Thread.current[:user]
Thread.current[:user] = user
- #Thread.current[:api_client_authorization] = ApiClientAuthorization.
- # where('user_id=? and scopes is null', user.id).
- # order('expires_at desc').
- # first
begin
yield
ensure
Thread.current[:user] = user_was
- #Thread.current[:api_client_authorization] = auth_was
end
end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'current_api_client'
-
-module SweepTrashedCollections
- extend CurrentApiClient
-
- def self.sweep_now
- act_as_system_user do
- Collection.
- where('delete_at is not null and delete_at < statement_timestamp()').
- destroy_all
- Collection.
- where('is_trashed = false and trash_at < statement_timestamp()').
- update_all('is_trashed = true')
- end
- end
-
- def self.sweep_if_stale
- return if Rails.configuration.trash_sweep_interval <= 0
- exp = Rails.configuration.trash_sweep_interval.seconds
- need = false
- Rails.cache.fetch('SweepTrashedCollections', expires_in: exp) do
- need = true
- end
- if need
- Thread.new do
- Thread.current.abort_on_exception = false
- begin
- sweep_now
- rescue => e
- Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
- ensure
- ActiveRecord::Base.connection.close
- end
- end
- end
- end
-end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'current_api_client'
+
+module SweepTrashedObjects
+ extend CurrentApiClient
+
+ def self.delete_project_and_contents(p_uuid)
+ p = Group.find_by_uuid(p_uuid)
+ if !p || p.group_class != 'project'
+ raise "can't sweep group '#{p_uuid}', it may not exist or not be a project"
+ end
+ # First delete sub projects
+ Group.where({group_class: 'project', owner_uuid: p_uuid}).each do |sub_project|
+ delete_project_and_contents(sub_project.uuid)
+ end
+ # Next, iterate over all tables which have owner_uuid fields, with some
+ # exceptions, and delete records owned by this project
+ skipped_classes = ['Group', 'User']
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
+ if !skipped_classes.include?(klass.name) && klass.columns.collect(&:name).include?('owner_uuid')
+ klass.where({owner_uuid: p_uuid}).destroy_all
+ end
+ end
+ # Finally delete the project itself
+ p.destroy
+ end
+
+ def self.sweep_now
+ act_as_system_user do
+ # Sweep trashed collections
+ Collection.
+ where('delete_at is not null and delete_at < statement_timestamp()').
+ destroy_all
+ Collection.
+ where('is_trashed = false and trash_at < statement_timestamp()').
+ update_all('is_trashed = true')
+
+ # Sweep trashed projects and their contents
+ Group.
+ where({group_class: 'project'}).
+ where('delete_at is not null and delete_at < statement_timestamp()').each do |project|
+ delete_project_and_contents(project.uuid)
+ end
+ Group.
+ where({group_class: 'project'}).
+ where('is_trashed = false and trash_at < statement_timestamp()').
+ update_all('is_trashed = true')
+ end
+ end
+
+ def self.sweep_if_stale
+ return if Rails.configuration.trash_sweep_interval <= 0
+ exp = Rails.configuration.trash_sweep_interval.seconds
+ need = false
+ Rails.cache.fetch('SweepTrashedObjects', expires_in: exp) do
+ need = true
+ end
+ if need
+ Thread.new do
+ Thread.current.abort_on_exception = false
+ begin
+ sweep_now
+ rescue => e
+ Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
+ ensure
+ ActiveRecord::Base.connection.close
+ end
+ end
+ end
+ end
+end
Get an existing anonymous user token. If no such token exists \
or if this option is omitted, a new token is created and returned.
eos
+ opt :token, "token to create (optional)", :type => :string
end
get_existing = opts[:get]
+supplied_token = opts[:token]
require File.dirname(__FILE__) + '/../config/environment'
include ApplicationHelper
act_as_system_user
-def create_api_client_auth
+def create_api_client_auth(supplied_token=nil)
+
+ # If token is supplied, verify that it indeed is a superuser token
+ if supplied_token
+ api_client_auth = ApiClientAuthorization.
+ where(api_token: supplied_token).
+ first
+ if !api_client_auth
+ # fall through to create a token
+ else
+ raise "Token exists, aborting!"
+ end
+ end
+
api_client_auth = ApiClientAuthorization.
new(user: anonymous_user,
api_client_id: 0,
expires_at: Time.now + 100.years,
- scopes: ['GET /'])
+ scopes: ['GET /'],
+ api_token: supplied_token)
api_client_auth.save!
api_client_auth.reload
+ api_client_auth
end
if get_existing
# either not a get or no api_client_auth was found
if !api_client_auth
- api_client_auth = create_api_client_auth
+ api_client_auth = create_api_client_auth(supplied_token)
end
# print it to the console
name: trashed project
group_class: project
trash_at: 2001-01-01T00:00:00Z
- delete_at: 2038-03-01T00:00:00Z
+ delete_at: 2008-03-01T00:00:00Z
is_trashed: true
modified_at: 2001-01-01T00:00:00Z
trash_at: 2001-01-01T00:00:00Z
delete_at: 2038-03-01T00:00:00Z
is_trashed: true
+ modified_at: 2001-01-01T00:00:00Z
+
+trashed_on_next_sweep:
+ uuid: zzzzz-j7d0g-soontobetrashed
+ owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+ name: soon to be trashed project
+ group_class: project
+ trash_at: 2001-01-01T00:00:00Z
+ delete_at: 2038-03-01T00:00:00Z
+ is_trashed: false
modified_at: 2001-01-01T00:00:00Z
\ No newline at end of file
state: Complete
script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
+job_in_trashed_project:
+ uuid: zzzzz-8i9sb-subprojectjob02
+ created_at: 2014-10-15 12:00:00
+ owner_uuid: zzzzz-j7d0g-trashedproject2
+ log: ~
+ repository: active/foo
+ script: hash
+ script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
+ state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
+
running_will_be_completed:
uuid: zzzzz-8i9sb-rshmckwoma9pjh8
owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
# SPDX-License-Identifier: AGPL-3.0
require 'test_helper'
-require 'sweep_trashed_collections'
+require 'sweep_trashed_objects'
class CollectionTest < ActiveSupport::TestCase
include DbCurrentTime
assert_includes(coll_uuids, collections(:docker_image).uuid)
end
- test "move to trash in SweepTrashedCollections" do
+ test "move collections to trash in SweepTrashedObjects" do
c = collections(:trashed_on_next_sweep)
refute_empty Collection.where('uuid=? and is_trashed=false', c.uuid)
assert_raises(ActiveRecord::RecordNotUnique) do
name: c.name)
end
end
- SweepTrashedCollections.sweep_now
+ SweepTrashedObjects.sweep_now
c = Collection.where('uuid=? and is_trashed=true', c.uuid).first
assert c
act_as_user users(:active) do
end
end
- test "delete in SweepTrashedCollections" do
+ test "delete collections in SweepTrashedObjects" do
uuid = 'zzzzz-4zz18-3u1p5umicfpqszp' # deleted_on_next_sweep
assert_not_empty Collection.where(uuid: uuid)
- SweepTrashedCollections.sweep_now
+ SweepTrashedObjects.sweep_now
assert_empty Collection.where(uuid: uuid)
end
- test "delete referring links in SweepTrashedCollections" do
+ test "delete referring links in SweepTrashedObjects" do
uuid = collections(:trashed_on_next_sweep).uuid
act_as_system_user do
Link.create!(head_uuid: uuid,
Collection.where(uuid: uuid).
update_all(is_trashed: true, trash_at: past, delete_at: past)
assert_not_empty Collection.where(uuid: uuid)
- SweepTrashedCollections.sweep_now
+ SweepTrashedObjects.sweep_now
assert_empty Collection.where(uuid: uuid)
end
end
test "Request is finalized when its container is cancelled" do
set_user_from_auth :active
cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 1)
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
act_as_system_user do
Container.find_by_uuid(cr.container_uuid).
cr.reload
assert_equal "Final", cr.state
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
end
test "Request is finalized when its container is completed" do
cr = create_minimal_req!(owner_uuid: project.uuid,
priority: 1,
state: "Committed")
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
c = act_as_system_user do
c = Container.find_by_uuid(cr.container_uuid)
cr.reload
assert_equal "Final", cr.state
+ assert_equal users(:active).uuid, cr.modified_by_user_uuid
['output', 'log'].each do |out_type|
pdh = Container.find_by_uuid(cr.container_uuid).send(out_type)
assert_equal(1, Collection.where(portable_data_hash: pdh,
cr2 = create_minimal_req!
cr2.update_attributes!(priority: 10, state: "Committed", requesting_container_uuid: c.uuid, command: ["echo", "foo2"], container_count_max: 1)
cr2.reload
+ assert_equal users(:active).uuid, cr2.modified_by_user_uuid
c2 = Container.find_by_uuid cr2.container_uuid
assert_operator 0, :<, c2.priority
cr2.reload
assert_equal 0, cr2.priority
+ assert_equal users(:active).uuid, cr2.modified_by_user_uuid
c2.reload
assert_equal 0, c2.priority
assert g_foo.errors.messages[:owner_uuid].join(" ").match(/ownership cycle/)
end
- test "delete group hides contents" do
+ test "trash group hides contents" do
set_user_from_auth :active_trustedclient
g_foo = Group.create!(name: "foo")
assert Collection.readable_by(users(:active)).where(uuid: col.uuid).any?
end
- test "delete group" do
+ test "trash group" do
set_user_from_auth :active_trustedclient
g_foo = Group.create!(name: "foo")
end
- test "delete subgroup" do
+ test "trash subgroup" do
set_user_from_auth :active_trustedclient
g_foo = Group.create!(name: "foo")
assert Group.readable_by(users(:active), {:include_trash => true}).where(uuid: g_baz.uuid).any?
end
- test "delete subsubgroup" do
+ test "trash subsubgroup" do
set_user_from_auth :active_trustedclient
g_foo = Group.create!(name: "foo")
end
- test "delete group propagates to subgroups" do
+ test "trash group propagates to subgroups" do
set_user_from_auth :active_trustedclient
g_foo = groups(:trashed_project)
assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
assert Collection.readable_by(users(:active)).where(uuid: col.uuid).any?
- # this one should still be deleted.
+ # this one should still be trashed.
assert Group.readable_by(users(:active)).where(uuid: g_baz.uuid).empty?
g_baz.update! is_trashed: false
assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
end
+ test "move projects to trash in SweepTrashedObjects" do
+ p = groups(:trashed_on_next_sweep)
+ assert_empty Group.where('uuid=? and is_trashed=true', p.uuid)
+ SweepTrashedObjects.sweep_now
+ assert_not_empty Group.where('uuid=? and is_trashed=true', p.uuid)
+ end
+
+ test "delete projects and their contents in SweepTrashedObjects" do
+ g_foo = groups(:trashed_project)
+ g_bar = groups(:trashed_subproject)
+ g_baz = groups(:trashed_subproject3)
+ col = collections(:collection_in_trashed_subproject)
+ job = jobs(:job_in_trashed_project)
+ cr = container_requests(:cr_in_trashed_project)
+ # Save how many objects were before the sweep
+ user_nr_was = User.all.length
+ coll_nr_was = Collection.all.length
+ group_nr_was = Group.where('group_class<>?', 'project').length
+ project_nr_was = Group.where(group_class: 'project').length
+ cr_nr_was = ContainerRequest.all.length
+ job_nr_was = Job.all.length
+ assert_not_empty Group.where(uuid: g_foo.uuid)
+ assert_not_empty Group.where(uuid: g_bar.uuid)
+ assert_not_empty Group.where(uuid: g_baz.uuid)
+ assert_not_empty Collection.where(uuid: col.uuid)
+ assert_not_empty Job.where(uuid: job.uuid)
+ assert_not_empty ContainerRequest.where(uuid: cr.uuid)
+ SweepTrashedObjects.sweep_now
+ assert_empty Group.where(uuid: g_foo.uuid)
+ assert_empty Group.where(uuid: g_bar.uuid)
+ assert_empty Group.where(uuid: g_baz.uuid)
+ assert_empty Collection.where(uuid: col.uuid)
+ assert_empty Job.where(uuid: job.uuid)
+ assert_empty ContainerRequest.where(uuid: cr.uuid)
+ # No unwanted deletions should have happened
+ assert_equal user_nr_was, User.all.length
+ assert_equal coll_nr_was-2, # collection_in_trashed_subproject
+ Collection.all.length # & deleted_on_next_sweep collections
+ assert_equal group_nr_was, Group.where('group_class<>?', 'project').length
+ assert_equal project_nr_was-3, Group.where(group_class: 'project').length
+ assert_equal cr_nr_was-1, ContainerRequest.all.length
+ assert_equal job_nr_was-1, Job.all.length
+ end
end
{"object_uuid", "=", container.UUID},
{"event_type", "=", "dispatch"},
}}, &ll)
+ c.Assert(err, IsNil)
c.Assert(len(ll.Items), Equals, 1)
}
cr := NewContainerRunner(&ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
_, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ c.Check(err, IsNil)
_, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
c.Check(err, NotNil)
for _, v := range []string{"p1", "p2", "p3", "p4", "p5"} {
info, err := os.Lstat(realTemp + "/" + v)
+ c.Assert(err, IsNil)
_, _, _, err = cr.derefOutputSymlink(realTemp+"/"+v, info)
c.Assert(err, NotNil)
}
include agpl-3.0.txt
include arvados-docker-cleaner.service
+include arvados_version.py
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+
+def git_latest_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
+def git_timestamp_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+
+def save_version(setup_dir, module, v):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+ except subprocess.CalledProcessError:
+ pass
+
+ return read_version(setup_dir, module)
#
# SPDX-License-Identifier: AGPL-3.0
+from __future__ import absolute_import
import os
import sys
-import setuptools.command.egg_info as egg_info_cmd
+import re
from setuptools import setup, find_packages
-tagger = egg_info_cmd.egg_info
-version = os.environ.get("ARVADOS_BUILDING_VERSION")
-if not version:
- version = "0.1"
- try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
- except ImportError:
- pass
+SETUP_DIR = os.path.dirname(__file__) or '.'
+README = os.path.join(SETUP_DIR, 'README.rst')
+
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvados_docker")
+
+short_tests_only = False
+if '--short-tests-only' in sys.argv:
+ short_tests_only = True
+ sys.argv.remove('--short-tests-only')
setup(name="arvados-docker-cleaner",
version=version,
'mock',
],
test_suite='tests',
- zip_safe=False,
- cmdclass={'egg_info': tagger},
+ zip_safe=False
)
include agpl-3.0.txt
include README.rst
+include arvados_version.py
\ No newline at end of file
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-import pkg_resources
-
-__version__ = pkg_resources.require('arvados_fuse')[0].version
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+
+def git_latest_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
+def git_timestamp_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+
+def save_version(setup_dir, module, v):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+ except subprocess.CalledProcessError:
+ pass
+
+ return read_version(setup_dir, module)
#
# SPDX-License-Identifier: AGPL-3.0
+from __future__ import absolute_import
import os
import sys
-import setuptools.command.egg_info as egg_info_cmd
+import re
from setuptools import setup, find_packages
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-tagger = egg_info_cmd.egg_info
-version = os.environ.get("ARVADOS_BUILDING_VERSION")
-if not version:
- version = "0.1"
- try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
- except ImportError:
- pass
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvados_fuse")
short_tests_only = False
if '--short-tests-only' in sys.argv:
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
- zip_safe=False,
- cmdclass={'egg_info': tagger},
+ zip_safe=False
)
// Discard (but measure size of) anything past 128 MiB.
var discarded int64
if err == io.ErrUnexpectedEOF {
- err = nil
buf = buf[:n]
} else {
c.Assert(err, check.Equals, nil)
if !fs.writing {
return
}
- dir, name := path.Split(name)
+ dir, _ := path.Split(name)
if dir == "" || dir == "/" {
return
}
// Now try to put the block through
if locatorIn == "" {
- if bytes, err := ioutil.ReadAll(req.Body); err != nil {
- err = errors.New(fmt.Sprintf("Error reading request body: %s", err))
+ bytes, err2 := ioutil.ReadAll(req.Body)
+ if err2 != nil {
+ _ = errors.New(fmt.Sprintf("Error reading request body: %s", err2))
status = http.StatusInternalServerError
return
- } else {
- locatorOut, wroteReplicas, err = kc.PutB(bytes)
}
+ locatorOut, wroteReplicas, err = kc.PutB(bytes)
} else {
locatorOut, wroteReplicas, err = kc.PutHR(locatorIn, req.Body, expectLength)
}
req, err := http.NewRequest("POST",
"http://"+listener.Addr().String()+"/",
strings.NewReader("TestViaHeader"))
+ c.Assert(err, Equals, nil)
req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
resp, err := (&http.Client{}).Do(req)
c.Assert(err, Equals, nil)
reader, blocklen, _, err := kc.Get(hash2)
c.Assert(err, Equals, nil)
all, err := ioutil.ReadAll(reader)
+ c.Check(err, IsNil)
c.Check(all, DeepEquals, []byte("foo"))
c.Check(blocklen, Equals, int64(3))
c.Log("Finished Get (expected success)")
reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
c.Assert(err, Equals, nil)
all, err := ioutil.ReadAll(reader)
+ c.Check(err, IsNil)
c.Check(all, DeepEquals, []byte(""))
c.Check(blocklen, Equals, int64(0))
c.Log("Finished Get zero block")
req, err := http.NewRequest("OPTIONS",
fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
nil)
+ c.Assert(err, IsNil)
req.Header.Add("Access-Control-Request-Method", "PUT")
req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
resp, err := client.Do(req)
c.Check(err, Equals, nil)
c.Check(resp.StatusCode, Equals, 200)
body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, IsNil)
c.Check(string(body), Equals, "")
c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
req, err := http.NewRequest("POST",
"http://"+listener.Addr().String()+"/",
strings.NewReader("qux"))
+ c.Check(err, IsNil)
req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
req.Header.Add("Content-Type", "application/octet-stream")
resp, err := client.Do(req)
c.Check(err, Equals, nil)
reader, blocklen, _, err := kc.Get(hash)
- c.Assert(err, Equals, nil)
+ c.Assert(err, IsNil)
c.Check(blocklen, Equals, int64(10))
all, err := ioutil.ReadAll(reader)
+ c.Assert(err, IsNil)
c.Check(all, DeepEquals, data)
// Put some more blocks
- _, rep, err = kc.PutB([]byte("some-more-index-data"))
- c.Check(err, Equals, nil)
+ _, _, err = kc.PutB([]byte("some-more-index-data"))
+ c.Check(err, IsNil)
kc.Arvados.ApiToken = arvadostest.DataManagerToken
}
// Call Trash, then check canTrash and canGetAfterTrash
- loc, blk = setupScenario()
+ loc, _ = setupScenario()
err = v.Trash(loc)
c.Check(err == nil, check.Equals, scenario.canTrash)
_, err = v.Get(context.Background(), loc, buf)
}
// Call Untrash, then check canUntrash
- loc, blk = setupScenario()
+ loc, _ = setupScenario()
err = v.Untrash(loc)
c.Check(err == nil, check.Equals, scenario.canUntrash)
if scenario.dataT != none || scenario.trashT != none {
// Call EmptyTrash, then check haveTrashAfterEmpty and
// freshAfterEmpty
- loc, blk = setupScenario()
+ loc, _ = setupScenario()
v.EmptyTrash()
_, err = v.bucket.Head("trash/"+loc, nil)
c.Check(err == nil, check.Equals, scenario.haveTrashAfterEmpty)
// goes away.
// (In Azure volumes, un/trash changes Mtime, so first backdate again)
v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
- err = v.Trash(TestHash)
+ _ = v.Trash(TestHash)
err = checkGet()
if err == nil || !os.IsNotExist(err) {
t.Fatalf("os.IsNotExist(%v) should have been true", err)
"+", fileInfo[0].Size(),
" ", fileInfo[0].ModTime().UnixNano(),
"\n")
+ if err != nil {
+ log.Print("Error writing : ", err)
+ lastErr = err
+ break
+ }
}
blockdir.Close()
}
include agpl-3.0.txt
include README.rst
+include arvados_version.py
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+
+def git_latest_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
+def git_timestamp_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+
+def save_version(setup_dir, module, v):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+ except subprocess.CalledProcessError:
+ pass
+
+ return read_version(setup_dir, module)
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-import pkg_resources
-
-__version__ = pkg_resources.require('arvados-node-manager')[0].version
import pykka
+from .status import tracker
+
class _TellCallableProxy(object):
"""Internal helper class for proxying callables."""
exception_type is OSError and exception_value.errno == errno.ENOMEM):
lg.critical("Unhandled exception is a fatal error, killing Node Manager")
self._killfunc(os.getpid(), signal.SIGKILL)
+ tracker.counter_add('actor_exceptions')
def ping(self):
return True
import time
from ..config import CLOUD_ERRORS
+from ..status import tracker
from libcloud.common.exceptions import BaseHTTPError, RateLimitReachedError
ARVADOS_TIMEFMT = '%Y-%m-%dT%H:%M:%SZ'
if error.code == 429 or error.code >= 500:
should_retry = True
except CLOUD_ERRORS as error:
+ tracker.counter_add('cloud_errors')
should_retry = True
except errors as error:
should_retry = True
# As a libcloud workaround for drivers that don't use
# typed exceptions, consider bare Exception() objects
# retryable.
- should_retry = type(error) is Exception
+ if type(error) is Exception:
+ tracker.counter_add('cloud_errors')
+ should_retry = True
else:
- # No exception,
+ # No exception
self.retry_wait = self.min_retry_wait
return ret
arvados_node_missing, RetryMixin
from ...clientactor import _notify_subscribers
from ... import config
+from ... import status
from .transitions import transitions
QuotaExceeded = "QuotaExceeded"
self.cancel_shutdown("No longer eligible for shut down because %s" % reason,
try_resume=True)
return
+ # If boot failed, count the event
+ if self._monitor.get_state().get() == 'unpaired':
+ status.tracker.counter_add('boot_failures')
self._destroy_node()
def _destroy_node(self):
#if state == 'idle' and self.arvados_node['job_uuid']:
# state = 'busy'
+ # Update idle node times tracker
+ if state == 'idle':
+ status.tracker.idle_in(self.arvados_node['hostname'])
+ else:
+ status.tracker.idle_out(self.arvados_node['hostname'])
+
return state
def in_state(self, *states):
from libcloud.compute.base import NodeDriver, NodeAuthSSHKey
from ...config import CLOUD_ERRORS
+from ...status import tracker
from .. import RetryMixin
class BaseComputeNodeDriver(RetryMixin):
def list_nodes(self, **kwargs):
l = self.list_kwargs.copy()
l.update(kwargs)
- return self.real.list_nodes(**l)
+ try:
+ return self.real.list_nodes(**l)
+ except CLOUD_ERRORS:
+ tracker.counter_add('list_nodes_errors')
+ raise
def create_cloud_name(self, arvados_node):
"""Return a cloud node name for the given Arvados node record.
try:
return self.search_for_now(kwargs['name'], 'list_nodes', self._name_key)
except ValueError:
+ tracker.counter_add('create_node_errors')
raise create_error
def post_create_node(self, cloud_node):
def destroy_node(self, cloud_node):
try:
return self.real.destroy_node(cloud_node)
- except CLOUD_ERRORS as destroy_error:
+ except CLOUD_ERRORS:
# Sometimes the destroy node request succeeds but times out and
# raises an exception instead of returning success. If this
# happens, we get a noisy stack trace. Check if the node is still
# it, which means destroy_node actually succeeded.
return True
# The node is still on the list. Re-raise.
+ tracker.counter_add('destroy_node_errors')
raise
# Now that we've defined all our own methods, delegate generic, public
if hasattr(record.cloud_node, "_nodemanager_recently_booted"):
self.cloud_nodes.add(record)
else:
- # Node disappeared from the cloud node list. Stop the monitor
- # actor if necessary and forget about the node.
+ # Node disappeared from the cloud node list. If it's paired,
+ # remove its idle time counter.
+ if record.arvados_node:
+ status.tracker.idle_out(record.arvados_node.get('hostname'))
+ # Stop the monitor actor if necessary and forget about the node.
if record.actor:
try:
record.actor.stop()
updates.setdefault('nodes_'+s, 0)
updates['nodes_'+s] += 1
updates['nodes_wish'] = len(self.last_wishlist)
+ updates['node_quota'] = self.node_quota
status.tracker.update(updates)
def _state_counts(self, size):
def update_server_wishlist(self, wishlist):
self._update_poll_time('server_wishlist')
- self.last_wishlist = wishlist
+ requestable_nodes = self.node_quota - (self._nodes_booting(None) + len(self.cloud_nodes))
+ self.last_wishlist = wishlist[:requestable_nodes]
for size in reversed(self.server_calculator.cloud_sizes):
try:
nodes_wanted = self._nodes_wanted(size)
self._later.start_node(size)
elif (nodes_wanted < 0) and self.booting:
self._later.stop_booting_node(size)
- except Exception as e:
+ except Exception:
self._logger.exception("while calculating nodes wanted for size %s", getattr(size, "id", "(id not available)"))
try:
self._update_tracker()
def _send_request(self):
queuelist = []
if self.slurm_queue:
- # cpus, memory, tempory disk space, reason, job name, feature constraints
- squeue_out = subprocess.check_output(["squeue", "--state=PENDING", "--noheader", "--format=%c|%m|%d|%r|%j|%f"])
+ # cpus, memory, tempory disk space, reason, job name, feature constraints, priority
+ squeue_out = subprocess.check_output(["squeue", "--state=PENDING", "--noheader", "--format=%c|%m|%d|%r|%j|%f|%Q"])
for out in squeue_out.splitlines():
try:
- cpu, ram, disk, reason, jobname, features = out.split("|", 5)
+ cpu, ram, disk, reason, jobname, features, priority = out.split("|", 6)
except ValueError:
self._logger.warning("ignored malformed line in squeue output: %r", out)
continue
"uuid": jobname,
"runtime_constraints": {
"instance_type": instance_type,
- }
+ },
+ "priority": int(priority)
})
break
else:
"min_cores_per_node": cpu,
"min_ram_mb_per_node": self.coerce_to_mb(ram),
"min_scratch_mb_per_node": self.coerce_to_mb(disk)
- }
+ },
+ "priority": int(priority)
})
+ queuelist.sort(key=lambda x: x.get('priority', 1), reverse=True)
if self.jobs_queue:
queuelist.extend(self._client.jobs().queue().execute()['items'])
from future import standard_library
import http.server
+import time
import json
import logging
import socketserver
return
self._config = config
self._tracker = tracker
+ self._tracker.update({'config_max_nodes': config.getint('Daemon', 'max_nodes')})
super(Server, self).__init__(
(config.get('Manage', 'address'), port), Handler)
self._thread = threading.Thread(target=self.serve_forever)
class Tracker(object):
def __init__(self):
self._mtx = threading.Lock()
- self._latest = {}
+ self._latest = {
+ 'list_nodes_errors': 0,
+ 'create_node_errors': 0,
+ 'destroy_node_errors': 0,
+ 'boot_failures': 0,
+ 'actor_exceptions': 0
+ }
self._version = {'Version' : __version__}
+ self._idle_nodes = {}
def get_json(self):
with self._mtx:
- return json.dumps(dict(self._latest, **self._version))
+ times = {'idle_times' : {}}
+ now = time.time()
+ for node, ts in self._idle_nodes.items():
+ times['idle_times'][node] = int(now - ts)
+ return json.dumps(
+ dict(dict(self._latest, **self._version), **times))
def keys(self):
with self._mtx:
return self._latest.keys()
+ def get(self, key):
+ with self._mtx:
+ return self._latest.get(key)
+
def update(self, updates):
with self._mtx:
self._latest.update(updates)
+ def counter_add(self, counter, value=1):
+ with self._mtx:
+ self._latest.setdefault(counter, 0)
+ self._latest[counter] += value
+
+ def idle_in(self, nodename):
+ with self._mtx:
+ if self._idle_nodes.get(nodename):
+ return
+ self._idle_nodes[nodename] = time.time()
+
+ def idle_out(self, nodename):
+ with self._mtx:
+ try:
+ del self._idle_nodes[nodename]
+ except KeyError:
+ pass
tracker = Tracker()
#
# SPDX-License-Identifier: AGPL-3.0
+from __future__ import absolute_import
import os
import sys
-import setuptools.command.egg_info as egg_info_cmd
+import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or "."
+SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-tagger = egg_info_cmd.egg_info
-version = os.environ.get("ARVADOS_BUILDING_VERSION")
-if not version:
- version = "0.1"
- try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
- except ImportError:
- pass
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvnodeman")
+
+short_tests_only = False
+if '--short-tests-only' in sys.argv:
+ short_tests_only = True
+ sys.argv.remove('--short-tests-only')
setup(name='arvados-node-manager',
version=version,
'mock>=1.0',
'apache-libcloud>=2.3',
],
- zip_safe=False,
- cmdclass={'egg_info': tagger},
+ zip_safe=False
)
def set_squeue(g):
global all_jobs
update_script(os.path.join(fake_slurm, "squeue"), "#!/bin/sh\n" +
- "\n".join("echo '1|100|100|%s|%s|(null)'" % (v, k) for k,v in all_jobs.items()))
+ "\n".join("echo '1|100|100|%s|%s|(null)|1234567890'" % (v, k) for k,v in all_jobs.items()))
return 0
def set_queue_unsatisfiable(g):
global all_jobs, unsatisfiable_job_scancelled
# Simulate a job requesting a 99 core node.
update_script(os.path.join(fake_slurm, "squeue"), "#!/bin/sh\n" +
- "\n".join("echo '99|100|100|%s|%s|(null)'" % (v, k) for k,v in all_jobs.items()))
+ "\n".join("echo '99|100|100|%s|%s|(null)|1234567890'" % (v, k) for k,v in all_jobs.items()))
update_script(os.path.join(fake_slurm, "scancel"), "#!/bin/sh\n" +
"\ntouch %s" % unsatisfiable_job_scancelled)
return 0
# Provider
"azure"),
"test_single_node_azure": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", noop),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
- ], {
+ ],
+ # Checks (things that shouldn't happen)
+ {
r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 1),
r".*Setting node quota.*": fail,
},
+ # Driver class
"arvnodeman.test.fake_driver.FakeDriver",
+ # Jobs
{"34t0i-dz642-h42bg3hq4bdfpf9": "ReqNodeNotAvail"},
+ # Provider
"azure"),
"test_multiple_nodes": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
- ], {
+ ],
+ # Checks (things that shouldn't happen)
+ {
r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 4),
r".*Setting node quota.*": fail,
},
+ # Driver class
"arvnodeman.test.fake_driver.FakeDriver",
+ # Jobs
{"34t0i-dz642-h42bg3hq4bdfpf1": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf2": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf3": "ReqNodeNotAvail",
- "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"
- }, "azure"),
+ "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"},
+ # Provider
+ "azure"),
"test_hit_quota": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", remaining_jobs),
- (r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown)
- ], {
+ ],
+ # Checks (things that shouldn't happen)
+ {
r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 2),
r".*Sending create_node request.*": partial(expect_count, 5)
},
+ # Driver class
"arvnodeman.test.fake_driver.QuotaDriver",
+ # Jobs
{"34t0i-dz642-h42bg3hq4bdfpf1": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf2": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf3": "ReqNodeNotAvail",
- "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"
- }, "azure"),
+ "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"},
+ # Provider
+ "azure"),
"test_probe_quota": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", remaining_jobs),
- (r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*sending request", jobs_req),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
- ], {
+ ],
+ # Checks (things that shouldn't happen)
+ {
r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 6),
r".*Sending create_node request.*": partial(expect_count, 9)
},
+ # Driver class
"arvnodeman.test.fake_driver.QuotaDriver",
+ # Jobs
{"34t0i-dz642-h42bg3hq4bdfpf1": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf2": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf3": "ReqNodeNotAvail",
- "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"
- }, "azure"),
+ "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"},
+ # Provider
+ "azure"),
"test_no_hang_failing_node_create": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Client error: nope", noop),
(r".*Client error: nope", noop),
(r".*Client error: nope", noop),
],
+ # Checks (things that shouldn't happen)
{},
+ # Driver class
"arvnodeman.test.fake_driver.FailingDriver",
+ # Jobs
{"34t0i-dz642-h42bg3hq4bdfpf1": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf2": "ReqNodeNotAvail",
"34t0i-dz642-h42bg3hq4bdfpf3": "ReqNodeNotAvail",
- "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"
- }, "azure"),
+ "34t0i-dz642-h42bg3hq4bdfpf4": "ReqNodeNotAvail"},
+ # Provider
+ "azure"),
"test_retry_create": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Rate limit exceeded - scheduling retry in 12 seconds", noop),
(r".*Rate limit exceeded - scheduling retry in 2 seconds", noop),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", noop),
],
+ # Checks (things that shouldn't happen)
{},
+ # Driver class
"arvnodeman.test.fake_driver.RetryDriver",
- {"34t0i-dz642-h42bg3hq4bdfpf1": "ReqNodeNotAvail"
- }, "azure"),
+ # Jobs
+ {"34t0i-dz642-h42bg3hq4bdfpf1": "ReqNodeNotAvail"},
+ # Provider
+ "azure"),
"test_single_node_aws": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", noop),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
- ], {
+ ],
+ # Checks (things that shouldn't happen)
+ {
r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 1),
r".*Setting node quota.*": fail,
},
+ # Driver class
"arvnodeman.test.fake_driver.FakeAwsDriver",
+ # Jobs
{"34t0i-dz642-h42bg3hq4bdfpf9": "ReqNodeNotAvail"},
+ # Provider
"ec2"),
"test_single_node_gce": (
+ # Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", node_paired),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Not eligible for shut down because node state is \('busy', 'open', .*\)", node_busy),
(r".*ComputeNodeMonitorActor\..*\.([^[]*).*Suggesting shutdown because node state is \('idle', 'open', .*\)", noop),
(r".*ComputeNodeShutdownActor\..*\.([^[]*).*Shutdown success", node_shutdown),
- ], {
+ ],
+ # Checks (things that shouldn't happen)
+ {
r".*Suggesting shutdown because node state is \('down', .*\)": fail,
r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)": partial(expect_count, 1),
r".*Setting node quota.*": fail,
},
+ # Driver class
"arvnodeman.test.fake_driver.FakeGceDriver",
+ # Jobs
{"34t0i-dz642-h42bg3hq4bdfpf9": "ReqNodeNotAvail"},
+ # Provider
"gce")
}
from libcloud.common.exceptions import BaseHTTPError
import arvnodeman.computenode.dispatch as dispatch
+import arvnodeman.status as status
from arvnodeman.computenode.driver import BaseComputeNodeDriver
from . import testutil
def check_success_flag(self, expected, allow_msg_count=1):
# allow_msg_count is the number of internal messages that may
# need to be handled for shutdown to finish.
- for try_num in range(1 + allow_msg_count):
+ for _ in range(1 + allow_msg_count):
last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
if last_flag is expected:
break
else:
self.fail("success flag {} is not {}".format(last_flag, expected))
+ def test_boot_failure_counting(self, *mocks):
+ # A boot failure happens when a node transitions from unpaired to shutdown
+ status.tracker.update({'boot_failures': 0})
+ self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="unpaired"))
+ self.cloud_client.destroy_node.return_value = True
+ self.make_actor(cancellable=False)
+ self.check_success_flag(True, 2)
+ self.assertTrue(self.cloud_client.destroy_node.called)
+ self.assertEqual(1, status.tracker.get('boot_failures'))
+
def test_cancellable_shutdown(self, *mocks):
self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
self.cloud_client.destroy_node.return_value = True
self.assertFalse(self.cloud_client.destroy_node.called)
def test_uncancellable_shutdown(self, *mocks):
+ status.tracker.update({'boot_failures': 0})
self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
self.cloud_client.destroy_node.return_value = True
self.make_actor(cancellable=False)
self.check_success_flag(True, 4)
self.assertTrue(self.cloud_client.destroy_node.called)
+ # A normal shutdown shouldn't be counted as boot failure
+ self.assertEqual(0, status.tracker.get('boot_failures'))
def test_arvados_node_cleaned_after_shutdown(self, *mocks):
if len(mocks) == 1:
self.assertTrue(self.node_state('down'))
def test_in_idle_state(self):
+ idle_nodes_before = status.tracker._idle_nodes.keys()
self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
self.assertTrue(self.node_state('idle'))
self.assertFalse(self.node_state('busy'))
self.assertTrue(self.node_state('idle', 'busy'))
+ idle_nodes_after = status.tracker._idle_nodes.keys()
+ new_idle_nodes = [n for n in idle_nodes_after if n not in idle_nodes_before]
+ # There should be 1 additional idle node
+ self.assertEqual(1, len(new_idle_nodes))
def test_in_busy_state(self):
+ idle_nodes_before = status.tracker._idle_nodes.keys()
self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
self.assertFalse(self.node_state('idle'))
self.assertTrue(self.node_state('busy'))
self.assertTrue(self.node_state('idle', 'busy'))
+ idle_nodes_after = status.tracker._idle_nodes.keys()
+ new_idle_nodes = [n for n in idle_nodes_after if n not in idle_nodes_before]
+ # There shouldn't be any additional idle node
+ self.assertEqual(0, len(new_idle_nodes))
def test_init_shutdown_scheduling(self):
self.make_actor()
import mock
import arvnodeman.computenode.driver as driver_base
+import arvnodeman.status as status
+import arvnodeman.config as config
from . import testutil
class ComputeNodeDriverTestCase(unittest.TestCase):
self.assertIs(driver.search_for('id_1', 'list_images'),
driver.search_for('id_1', 'list_images'))
self.assertEqual(1, self.driver_mock().list_images.call_count)
+
+
+ class TestBaseComputeNodeDriver(driver_base.BaseComputeNodeDriver):
+ def arvados_create_kwargs(self, size, arvados_node):
+ return {'name': arvados_node}
+
+
+ def test_create_node_only_cloud_errors_are_counted(self):
+ status.tracker.update({'create_node_errors': 0})
+ errors = [(config.CLOUD_ERRORS[0], True), (KeyError, False)]
+ self.driver_mock().list_images.return_value = []
+ driver = self.TestBaseComputeNodeDriver({}, {}, {}, self.driver_mock)
+ error_count = 0
+ for an_error, is_cloud_error in errors:
+ self.driver_mock().create_node.side_effect = an_error
+ with self.assertRaises(an_error):
+ driver.create_node('1', 'id_1')
+ if is_cloud_error:
+ error_count += 1
+ self.assertEqual(error_count, status.tracker.get('create_node_errors'))
+
+ def test_list_nodes_only_cloud_errors_are_counted(self):
+ status.tracker.update({'list_nodes_errors': 0})
+ errors = [(config.CLOUD_ERRORS[0], True), (KeyError, False)]
+ driver = self.TestBaseComputeNodeDriver({}, {}, {}, self.driver_mock)
+ error_count = 0
+ for an_error, is_cloud_error in errors:
+ self.driver_mock().list_nodes.side_effect = an_error
+ with self.assertRaises(an_error):
+ driver.list_nodes()
+ if is_cloud_error:
+ error_count += 1
+ self.assertEqual(error_count, status.tracker.get('list_nodes_errors'))
+
+ def test_destroy_node_only_cloud_errors_are_counted(self):
+ status.tracker.update({'destroy_node_errors': 0})
+ errors = [(config.CLOUD_ERRORS[0], True), (KeyError, False)]
+ self.driver_mock().list_nodes.return_value = [testutil.MockSize(1)]
+ driver = self.TestBaseComputeNodeDriver({}, {}, {}, self.driver_mock)
+ error_count = 0
+ for an_error, is_cloud_error in errors:
+ self.driver_mock().destroy_node.side_effect = an_error
+ with self.assertRaises(an_error):
+ driver.destroy_node(testutil.MockSize(1))
+ if is_cloud_error:
+ error_count += 1
+ self.assertEqual(error_count, status.tracker.get('destroy_node_errors'))
size = testutil.MockSize(1)
self.make_daemon(want_sizes=[size])
self.busywait(lambda: self.node_setup.start.called)
+ self.assertIn('node_quota', status.tracker._latest)
def check_monitors_arvados_nodes(self, *arv_nodes):
self.busywait(lambda: len(arv_nodes) == len(self.monitored_arvados_nodes()))
self.daemon.update_cloud_nodes([]).get(self.TIMEOUT)
self.busywait(lambda: 1 == self.last_shutdown.stop.call_count)
+ def test_idle_node_disappearing_clears_status_idle_time_counter(self):
+ size = testutil.MockSize(1)
+ status.tracker._idle_nodes = {}
+ cloud_nodes = [testutil.cloud_node_mock(1, size=size)]
+ arv_nodes = [testutil.arvados_node_mock(1, job_uuid=None)]
+ self.make_daemon(cloud_nodes, arv_nodes, [size])
+ self.busywait(lambda: 1 == self.paired_monitor_count())
+ for mon_ref in self.monitor_list():
+ monitor = mon_ref.proxy()
+ if monitor.cloud_node.get(self.TIMEOUT) is cloud_nodes[-1]:
+ break
+ else:
+ self.fail("monitor for idle node not found")
+ self.assertEqual(1, status.tracker.get('nodes_idle'))
+ hostname = monitor.arvados_node.get()['hostname']
+ self.assertIn(hostname, status.tracker._idle_nodes)
+ # Simulate the node disappearing from the cloud node list
+ self.daemon.update_cloud_nodes([]).get(self.TIMEOUT)
+ self.busywait(lambda: 0 == self.alive_monitor_count())
+ self.assertNotIn(hostname, status.tracker._idle_nodes)
+
def test_shutdown_actor_cleanup_copes_with_dead_actors(self):
self.make_daemon(cloud_nodes=[testutil.cloud_node_mock()])
self.assertEqual(1, self.alive_monitor_count())
big = testutil.MockSize(2)
avail_sizes = [(testutil.MockSize(1), {"cores":1}),
(testutil.MockSize(2), {"cores":2})]
- self.make_daemon(want_sizes=[small, small, small, big],
+ self.make_daemon(want_sizes=[small, small, big, small],
avail_sizes=avail_sizes, max_nodes=3)
# the daemon runs in another thread, so we need to wait and see
self.assertEqual(2, sizecounts[small.id])
self.assertEqual(1, sizecounts[big.id])
+ def test_wishlist_ordering(self):
+ # Check that big nodes aren't prioritized; since #12199 containers are
+ # scheduled on specific node sizes.
+ small = testutil.MockSize(1)
+ big = testutil.MockSize(2)
+ avail_sizes = [(testutil.MockSize(1), {"cores":1}),
+ (testutil.MockSize(2), {"cores":2})]
+ self.make_daemon(want_sizes=[small, small, small, big],
+ avail_sizes=avail_sizes, max_nodes=3)
+
+ # the daemon runs in another thread, so we need to wait and see
+ # if it does all the work we're expecting it to do before stopping it.
+ self.busywait(lambda: self.node_setup.start.call_count == 3)
+ booting = self.daemon.booting.get(self.TIMEOUT)
+ self.stop_proxy(self.daemon)
+ sizecounts = {a[0].id: 0 for a in avail_sizes}
+ for b in booting.itervalues():
+ sizecounts[b.cloud_size.get().id] += 1
+ self.assertEqual(3, sizecounts[small.id])
+ self.assertEqual(0, sizecounts[big.id])
+
def test_wishlist_reconfigure(self):
small = testutil.MockSize(1)
big = testutil.MockSize(2)
from . import testutil
import arvnodeman.baseactor
+import arvnodeman.status as status
class BogusActor(arvnodeman.baseactor.BaseNodeManagerActor):
def __init__(self, e, killfunc=None):
self.assertTrue(kill_mock.called)
def test_nonfatal_error(self):
+ status.tracker.update({'actor_exceptions': 0})
kill_mock = mock.Mock('os.kill')
act = BogusActor.start(OSError(errno.ENOENT, ""), killfunc=kill_mock).tell_proxy()
act.doStuff()
act.actor_ref.stop(block=True)
self.assertFalse(kill_mock.called)
+ self.assertEqual(1, status.tracker.get('actor_exceptions'))
class WatchdogActorTest(testutil.ActorTestMixin, unittest.TestCase):
def test_unsatisfiable_jobs(self, mock_squeue, mock_scancel):
job_uuid = 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'
container_uuid = 'yyyyy-dz642-yyyyyyyyyyyyyyy'
- mock_squeue.return_value = "1|1024|0|(Resources)|" + container_uuid + "|\n"
+ mock_squeue.return_value = "1|1024|0|(Resources)|" + container_uuid + "||1234567890\n"
self.build_monitor([{'items': [{'uuid': job_uuid}]}],
self.MockCalculatorUnsatisfiableJobs(), True, True)
@mock.patch("subprocess.check_output")
def test_squeue_server_list(self, mock_squeue):
- mock_squeue.return_value = """1|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzy|(null)
-2|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzz|(null)
+ mock_squeue.return_value = """1|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzy|(null)|1234567890
+2|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzz|(null)|1234567890
"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
@mock.patch("subprocess.check_output")
def test_squeue_server_list_suffix(self, mock_squeue):
- mock_squeue.return_value = """1|1024M|0|(ReqNodeNotAvail, UnavailableNodes:compute123)|zzzzz-dz642-zzzzzzzzzzzzzzy|(null)
-1|2G|0|(ReqNodeNotAvail)|zzzzz-dz642-zzzzzzzzzzzzzzz|(null)
+ mock_squeue.return_value = """1|1024M|0|(ReqNodeNotAvail, UnavailableNodes:compute123)|zzzzz-dz642-zzzzzzzzzzzzzzy|(null)|1234567890
+1|2G|0|(ReqNodeNotAvail)|zzzzz-dz642-zzzzzzzzzzzzzzz|(null)|1234567890
"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
@mock.patch("subprocess.check_output")
def test_squeue_server_list_instancetype_constraint(self, mock_squeue):
- mock_squeue.return_value = """1|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzy|instancetype=z2.test\n"""
+ mock_squeue.return_value = """1|1024|0|(Resources)|zzzzz-dz642-zzzzzzzzzzzzzzy|instancetype=z2.test|1234567890\n"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
[(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]),
True, True)
from future import standard_library
import json
+import mock
+import random
import requests
import unittest
self.assertEqual(n, resp['nodes_'+str(n)])
self.assertEqual(1, resp['nodes_1'])
self.assertIn('Version', resp)
+ self.assertIn('config_max_nodes', resp)
+
+ def test_counters(self):
+ with TestServer() as srv:
+ resp = srv.get_status()
+ # Test counters existance
+ for counter in ['list_nodes_errors', 'create_node_errors',
+ 'destroy_node_errors', 'boot_failures', 'actor_exceptions']:
+ self.assertIn(counter, resp)
+ # Test counter increment
+ for count in range(1, 3):
+ status.tracker.counter_add('a_counter')
+ resp = srv.get_status()
+ self.assertEqual(count, resp['a_counter'])
+
+ @mock.patch('time.time')
+ def test_idle_times(self, time_mock):
+ with TestServer() as srv:
+ resp = srv.get_status()
+ node_name = 'idle_compute{}'.format(random.randint(1, 1024))
+ self.assertIn('idle_times', resp)
+ # Test add an idle node
+ time_mock.return_value = 10
+ status.tracker.idle_in(node_name)
+ time_mock.return_value += 10
+ resp = srv.get_status()
+ self.assertEqual(10, resp['idle_times'][node_name])
+ # Test adding the same idle node a 2nd time
+ time_mock.return_value += 10
+ status.tracker.idle_in(node_name)
+ time_mock.return_value += 10
+ resp = srv.get_status()
+ # Idle timestamp doesn't get reset if already exists
+ self.assertEqual(30, resp['idle_times'][node_name])
+ # Test remove idle node
+ status.tracker.idle_out(node_name)
+ resp = srv.get_status()
+ self.assertNotIn(node_name, resp['idle_times'])
class StatusServerDisabled(unittest.TestCase):
include agpl-3.0.txt
include crunchstat_summary/dygraphs.js
include crunchstat_summary/synchronizer.js
+include arvados_version.py
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+
+def git_latest_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'describe', '--abbrev=0']).strip()
+ return str(gitinfo.decode('utf-8'))
+
+def git_timestamp_tag():
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', '.']).strip()
+ return str(time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))))
+
+def save_version(setup_dir, module, v):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'w') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'r') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_latest_tag() + git_timestamp_tag())
+ except subprocess.CalledProcessError:
+ pass
+
+ return read_version(setup_dir, module)
#
# SPDX-License-Identifier: AGPL-3.0
+from __future__ import absolute_import
import os
import sys
-import setuptools.command.egg_info as egg_info_cmd
+import re
from setuptools import setup, find_packages
SETUP_DIR = os.path.dirname(__file__) or '.'
+README = os.path.join(SETUP_DIR, 'README.rst')
-tagger = egg_info_cmd.egg_info
-version = os.environ.get("ARVADOS_BUILDING_VERSION")
-if not version:
- version = "0.1"
- try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
- except ImportError:
- pass
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "crunchstat_summary")
+
+short_tests_only = False
+if '--short-tests-only' in sys.argv:
+ short_tests_only = True
+ sys.argv.remove('--short-tests-only')
setup(name='crunchstat_summary',
version=version,
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0'],
- zip_safe=False,
- cmdclass={'egg_info': tagger},
+ zip_safe=False
)
projects / arvados.git / commitdiff