taskid: v11,
node: node,
slot: slot,
- message: message,
+ message: message.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'),
type: type
});
count += 1;
redirect_to @object
end
+ def arv_normalize mt, *opts
+ r = ""
+ IO.popen(['arv-normalize'] + opts, 'w+b') do |io|
+ io.write mt
+ io.close_write
+ while buf = io.read(2**16)
+ r += buf
+ end
+ end
+ r
+ end
+
expose_action :combine_selected_files_into_collection do
lst = []
files = []
params["selection"].each do |s|
- m = CollectionsHelper.match(s)
+ a = ArvadosBase::resource_class_for_uuid s
+ m = nil
+ if a == Link
+ begin
+ m = CollectionsHelper.match(Link.find(s).head_uuid)
+ rescue
+ end
+ else
+ m = CollectionsHelper.match(s)
+ end
+
if m and m[1] and m[2]
lst.append(m[1] + m[2])
files.append(m)
files.each do |m|
mt = chash[m[1]+m[2]].manifest_text
if m[4]
- IO.popen(['arv-normalize', '--extract', m[4][1..-1]], 'w+b') do |io|
- io.write mt
- io.close_write
- while buf = io.read(2**20)
- combined += buf
- end
- end
+ combined += arv_normalize mt, '--extract', m[4][1..-1]
else
combined += chash[m[1]+m[2]].manifest_text
end
end
- normalized = ''
- IO.popen(['arv-normalize'], 'w+b') do |io|
- io.write combined
- io.close_write
- while buf = io.read(2**20)
- normalized += buf
- end
- end
+ normalized = arv_normalize combined
+ normalized_stripped = arv_normalize combined, '--strip'
require 'digest/md5'
d = Digest::MD5.new()
- d << normalized
- newuuid = "#{d.hexdigest}+#{normalized.length}"
+ d << normalized_stripped
+ newuuid = "#{d.hexdigest}+#{normalized_stripped.length}"
env = Hash[ENV].
merge({
})
IO.popen([env, 'arv-put', '--raw'], 'w+b') do |io|
- io.write normalized
+ io.write normalized_stripped
io.close_write
- while buf = io.read(2**20)
-
+ while buf = io.read(2**16)
end
end
def get_n_objects_of_class dataclass, size
@objects_map_for ||= {}
- raise ArgumentError, 'Argument is not a data class' unless dataclass.is_a? Class
+ raise ArgumentError, 'Argument is not a data class' unless dataclass.is_a? Class and dataclass < ArvadosBase
raise ArgumentError, 'Argument is not a valid limit size' unless (size && size>0)
# if the objects_map_for has a value for this dataclass, and the
selectables = []
attrtext = attrvalue
- if dataclass and dataclass.is_a? Class
+ if dataclass.is_a? Class and dataclass < ArvadosBase
objects = get_n_objects_of_class dataclass, 10
objects.each do |item|
items << item
render opts.merge(partial: "application/#{partial}")
end
end
-
+
def fa_icon_class_for_object object
case object.class.to_s.to_sym
when :User
end
end
+def arv_edit_save_tmp tmp
+ FileUtils::cp tmp.path, tmp.path + ".saved"
+ puts "Saved contents to " + tmp.path + ".saved"
+end
+
def arv_edit client, arvados, global_opts, remaining_opts
- n = remaining_opts.shift
- if n.nil? or n == "-h" or n == "--help"
+ uuid = remaining_opts.shift
+ if uuid.nil? or uuid == "-h" or uuid == "--help"
puts head_banner
puts "Usage: arv edit [uuid] [fields...]\n\n"
puts "Fetch the specified Arvados object, select the specified fields, \n"
# determine controller
- m = /([a-z0-9]{5})-([a-z0-9]{5})-([a-z0-9]{15})/.match n
+ m = /([a-z0-9]{5})-([a-z0-9]{5})-([a-z0-9]{15})/.match uuid
if !m
- if /^[a-f0-9]{32}/.match n
+ if /^[a-f0-9]{32}/.match uuid
abort "Arvados collections are not editable."
else
abort "#{n} does not appear to be an Arvados uuid"
api_method = 'arvados.' + rsc + '.get'
result = client.execute(:api_method => eval(api_method),
- :parameters => {"uuid" => n},
+ :parameters => {"uuid" => uuid},
:authenticated => false,
:headers => {
authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
require 'tempfile'
- tmp = Tempfile.new([n, "." + global_opts[:format]])
+ tmp = Tempfile.new([uuid, "." + global_opts[:format]])
tmp.write(content)
tmp.close
n += 1
end
puts "\nTry again (y/n)? "
- yn = $stdin.read 1
+ yn = "X"
+ while not ["y", "Y", "n", "N"].include?(yn)
+ yn = $stdin.read 1
+ end
if yn == 'n' or yn == 'N'
- exit 1
+ arv_edit_save_tmp tmp
+ abort
end
end
else
end
end
- tmp.close(true)
-
- if newobj != results
- api_method = 'arvados.' + rsc + '.update'
- dumped = Oj.dump(newobj)
- result = client.execute(:api_method => eval(api_method),
- :parameters => {"uuid" => n, rsc.singularize => dumped},
- :authenticated => false,
- :headers => {
- authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
- })
-
- begin
- results = JSON.parse result.body
- rescue JSON::ParserError => e
- abort "Failed to parse server response:\n" + e.to_s
- end
+ begin
+ if newobj != results
+ api_method = 'arvados.' + rsc + '.update'
+ dumped = Oj.dump(newobj)
- if result.response.status != 200
- puts "Update failed. Server responded #{result.response.status}: #{results['errors']} "
- puts "Update body was:"
- puts dumped
+ begin
+ result = client.execute(:api_method => eval(api_method),
+ :parameters => {"uuid" => uuid},
+ :body => { rsc.singularize => dumped },
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
+ rescue Exception => e
+ puts "Error communicating with server, error was #{e}"
+ puts "Update body was:"
+ puts dumped
+ arv_edit_save_tmp tmp
+ abort
+ end
+
+ begin
+ results = JSON.parse result.body
+ rescue JSON::ParserError => e
+ abort "Failed to parse server response:\n" + e.to_s
+ end
+
+ if result.response.status != 200
+ puts "Update failed. Server responded #{result.response.status}: #{results['errors']} "
+ puts "Update body was:"
+ puts dumped
+ arv_edit_save_tmp tmp
+ abort
+ end
+ else
+ puts "Object is unchanged, did not update."
end
- else
- puts "Object is unchanged, did not update."
+ ensure
+ tmp.close(true)
end
exit 0
:repository => c[:repository],
:nondeterministic => c[:nondeterministic],
:output_is_persistent => c[:output_is_persistent] || false,
+ :runtime_constraints => c[:runtime_constraints],
:owner_uuid => owner_uuid,
# TODO: Delete the following three attributes when
# supporting pre-20140418 API servers is no longer
ended += 1
if c[:job][:success] == true
succeeded += 1
- elsif c[:job][:success] == false
+ elsif c[:job][:success] == false or c[:job][:cancelled_at]
failed += 1
end
end
my $command =
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
."mkdir -p $ENV{JOB_WORK} $ENV{CRUNCH_TMP} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
+ ."&& chmod og+wrx $ENV{TASK_WORK}"
."&& cd $ENV{CRUNCH_TMP} ";
if ($build_script)
{
$command .=
q{$(ip -o address show scope global |
gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
- foreach my $env_key (qw(CRUNCH_SRC CRUNCH_TMP TASK_KEEPMOUNT))
- {
- $command .= "-v \Q$ENV{$env_key}:$ENV{$env_key}:rw\E ";
- }
+ $command .= "-v \Q$ENV{TASK_WORK}:/tmp/crunch-job:rw\E ";
+ $command .= "-v \Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
+ $command .= "-v \Q$ENV{TASK_KEEPMOUNT}:/mnt:ro\E ";
while (my ($env_key, $env_val) = each %ENV)
{
if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
- $command .= "-e \Q$env_key=$env_val\E ";
+ if ($env_key eq "TASK_WORK") {
+ $command .= "-e \QTASK_WORK=/tmp/crunch-job\E ";
+ }
+ elsif ($env_key eq "TASK_KEEPMOUNT") {
+ $command .= "-e \QTASK_KEEPMOUNT=/mnt\E ";
+ }
+ elsif ($env_key eq "CRUNCH_SRC") {
+ $command .= "-e \QCRUNCH_SRC=/tmp/crunch-src\E ";
+ }
+ else {
+ $command .= "-e \Q$env_key=$env_val\E ";
+ }
}
}
$command .= "\Q$docker_hash\E ";
+ $command .= "stdbuf -o0 -e0 ";
+ $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
} else {
- $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 "
+ # Non-docker run
+ $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
+ $command .= "stdbuf -o0 -e0 ";
+ $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
}
- $command .= "stdbuf -o0 -e0 ";
- $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
exit (111);
use Arvados;
$arv = Arvados->new(apiHost => 'arvados.local');
-
+
my $instances = $arv->{'pipeline_instances'}->{'list'}->execute();
print "UUID is ", $instances->{'items'}->[0]->{'uuid'}, "\n";
-
+
$uuid = 'eiv0u-arx5y-2c5ovx43zw90gvh';
$instance = $arv->{'pipeline_instances'}->{'get'}->execute('uuid' => $uuid);
print "ETag is ", $instance->{'etag'}, "\n";
-
+
$instance->{'active'} = 1;
$instance->{'name'} = '';
$instance->save();
=cut
package Arvados;
+
+use Net::SSL (); # From Crypt-SSLeay
+BEGIN {
+ $Net::HTTPS::SSL_SOCKET_CLASS = "Net::SSL"; # Force use of Net::SSL
+}
+
use JSON;
-use Data::Dumper;
-use IO::Socket::SSL;
use Carp;
use Arvados::ResourceAccessor;
use Arvados::ResourceMethod;
use Arvados::ResourceProxy;
use Arvados::ResourceProxyList;
use Arvados::Request;
+use Data::Dumper;
$Arvados::VERSION = 0.1;
$config = load_config_file("$ENV{HOME}/.config/arvados/settings.conf");
- $self->{'authToken'} ||=
+ $self->{'authToken'} ||=
$ENV{ARVADOS_API_TOKEN} || $config->{ARVADOS_API_TOKEN};
$self->{'apiHost'} ||=
$ENV{ARVADOS_API_HOST} || $config->{ARVADOS_API_HOST};
+ $self->{'noVerifyHostname'} ||=
+ $ENV{ARVADOS_API_HOST_INSECURE};
+
$self->{'apiProtocolScheme'} ||=
$ENV{ARVADOS_API_PROTOCOL_SCHEME} ||
$config->{ARVADOS_API_PROTOCOL_SCHEME};
{
my $self = shift;
local $ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'};
- if ($opts{'noVerifyHostname'} || ($host =~ /\.local$/)) {
+ if ($self->{'noVerifyHostname'} || ($host =~ /\.local$/)) {
$ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0;
}
Arvados::Request->new();
for f in s.all_files():
yield f
- def manifest_text(self):
+ def manifest_text(self, strip=False):
self._populate()
- return self._manifest_text
+ if strip:
+ m = ''.join([StreamReader(stream).manifest_text(strip=True) for stream in self._streams])
+ return m
+ else:
+ return self._manifest_text
class CollectionWriter(object):
KEEP_BLOCK_SIZE = 2**26
for x in fields[1:-1] ]
clean += fields[0] + ' ' + ' '.join(locators) + ' ' + fields[-1] + "\n"
return clean
-
+
def manifest_text(self):
self.finish_current_stream()
manifest = ''
block_size = data_locators[i][BLOCKSIZE]
block_start = data_locators[i][OFFSET]
block_end = block_start + block_size
-
+
while i < len(data_locators):
locator, block_size, block_start = data_locators[i]
block_end = block_start + block_size
dc = bz2.BZ2Decompressor()
return self.decompress(lambda segment: dc.decompress(segment), size)
elif re.search('\.gz$', self._name):
- dc = zlib.decompressobj(16+zlib.MAX_WBITS)
+ dc = zlib.decompressobj(16+zlib.MAX_WBITS)
return self.decompress(lambda segment: dc.decompress(dc.unconsumed_tail + segment), size)
else:
return self.readall(size)
self._keep = keep
else:
self._keep = Keep.global_client_object()
-
+
streamoffset = 0L
# parse stream
for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._data_locators, start, size):
data += self._keep.get(locator)[segmentoffset:segmentoffset+segmentsize]
return data
-
- def manifest_text(self):
+
+ def manifest_text(self, strip=False):
manifest_text = [self.name().replace(' ', '\\040')]
- manifest_text.extend([d[LOCATOR] for d in self._data_locators])
- manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
+ if strip:
+ for d in self._data_locators:
+ m = re.match(r'^[0-9a-f]{32}\+\d+', d[LOCATOR])
+ manifest_text.append(m.group(0))
+ else:
+ manifest_text.extend([d[LOCATOR] for d in self._data_locators])
+ manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
for seg in f.segments])
for f in self._files.values()])
return ' '.join(manifest_text) + '\n'
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', False)
p = subprocess.Popen(execargs, **kwargs)
- stdoutdata, stderrdata = p.communicate(None)
+ if kwargs['stdout'] == subprocess.PIPE:
+ stdoutdata, stderrdata = p.communicate(None)
+ else:
+ p.wait()
if p.returncode != 0:
raise errors.CommandFailedError(
"run_command %s exit %d:\n%s" %
break
zip_file.write(buf)
zip_file.close()
-
+
p = subprocess.Popen(["unzip",
"-q", "-o",
"-d", path,
description='Read manifest on standard input and put normalized manifest on standard output.')
parser.add_argument('--extract', type=str, help="The file to extract from the input manifest")
+parser.add_argument('--strip', action='store_true', help="Strip authorization tokens")
args = parser.parse_args()
import arvados
r = sys.stdin.read()
-
+
cr = arvados.CollectionReader(r)
if args.extract:
if fn in s.files():
sys.stdout.write(s.files()[fn].as_manifest())
else:
- sys.stdout.write(cr.manifest_text())
+ sys.stdout.write(cr.manifest_text(args.strip))
gem 'themes_for_rails'
-gem 'arvados-cli', '>= 0.1.20140328152103'
+gem 'arvados-cli', '>= 0.1.20140627084759'
# pg_power lets us use partial indexes in schema.rb in Rails 3
gem 'pg_power'
addressable (2.3.6)
andand (1.3.3)
arel (3.0.3)
- arvados (0.1.20140513131358)
+ arvados (0.1.20140627084759)
activesupport (>= 3.2.13)
andand
google-api-client (~> 0.6.3)
json (>= 1.7.7)
- arvados-cli (0.1.20140513131358)
+ arvados-cli (0.1.20140627084759)
activesupport (~> 3.2, >= 3.2.13)
andand (~> 1.3, >= 1.3.3)
arvados (~> 0.1.0)
DEPENDENCIES
acts_as_api
andand
- arvados-cli (>= 0.1.20140328152103)
+ arvados-cli (>= 0.1.20140627084759)
coffee-rails (~> 3.2.0)
database_cleaner
faye-websocket
#!/usr/bin/env ruby
+
if ENV["CRUNCH_DISPATCH_LOCKFILE"]
lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
lockfile = File.open(lockfilename, File::RDWR|File::CREAT, 0644)
require File.dirname(__FILE__) + '/../config/boot'
require File.dirname(__FILE__) + '/../config/environment'
-def cancel_stale_jobs
- Job.running.each do |jobrecord|
- f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
- if f
- age = (Time.now - f.created_at)
- if age > 300
- $stderr.puts "dispatch: failing orphan job #{jobrecord.uuid}, last log is #{age} seconds old"
- # job is marked running, but not known to crunch-dispatcher, and
- # hasn't produced any log entries for 5 minutes, so mark it as failed.
- jobrecord.running = false
- jobrecord.cancelled_at ||= Time.now
- jobrecord.finished_at ||= Time.now
- if jobrecord.success.nil?
- jobrecord.success = false
+class CancelJobs
+ include ApplicationHelper
+
+ def cancel_stale_jobs
+ act_as_system_user do
+ Job.running.each do |jobrecord|
+ f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
+ if f
+ age = (Time.now - f.created_at)
+ if age > 300
+ $stderr.puts "dispatch: failing orphan job #{jobrecord.uuid}, last log is #{age} seconds old"
+ # job is marked running, but not known to crunch-dispatcher, and
+ # hasn't produced any log entries for 5 minutes, so mark it as failed.
+ jobrecord.running = false
+ jobrecord.cancelled_at ||= Time.now
+ jobrecord.finished_at ||= Time.now
+ if jobrecord.success.nil?
+ jobrecord.success = false
+ end
+ jobrecord.save!
+ end
end
- jobrecord.save!
end
end
end
end
-cancel_stale_jobs
+CancelJobs.new.cancel_stale_jobs
cmd_args = nil
case Server::Application.config.crunch_job_wrapper
when :none
+ if @running.size > 0
+ # Don't run more than one at a time.
+ return
+ end
cmd_args = []
when :slurm_immediate
nodelist = nodes_available_for_job(job)
if op == "Read" {
disk[device].last_read = disk[device].next_read
disk[device].next_read = next
- if disk[device].last_read > 0 {
+ if disk[device].last_read > 0 && (disk[device].next_read != disk[device].last_read) {
stderr <- fmt.Sprintf("crunchstat: blkio.io_service_bytes %s read %v", device, disk[device].next_read-disk[device].last_read)
}
}
if op == "Write" {
disk[device].last_write = disk[device].next_write
disk[device].next_write = next
- if disk[device].last_write > 0 {
+ if disk[device].last_write > 0 && (disk[device].next_write != disk[device].last_write) {
stderr <- fmt.Sprintf("crunchstat: blkio.io_service_bytes %s write %v", device, disk[device].next_write-disk[device].last_write)
}
}
import re
import apiclient
import json
+import logging
from time import time
from llfuse import FUSEError
try:
self.update()
except apiclient.errors.HttpError as e:
- print e
+ logging.debug(e)
def __getitem__(self, item):
self.checkupdate()
cwd = cwd._entries[part]
for k, v in s.files().items():
cwd._entries[k] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v))
- print "found"
self.fresh()
+ return True
except Exception as detail:
- print("%s: error: %s" % (self.collection_locator,detail) )
+ logging.debug("arv-mount %s: error: %s" % (self.collection_locator,detail))
+ return False
class MagicDirectory(Directory):
'''A special directory that logically contains the set of all extant keep
if k in self._entries:
return True
try:
- if arvados.Keep.get(k):
+ e = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, k))
+ if e.update():
+ self._entries[k] = e
return True
else:
return False
except Exception as e:
- #print 'exception keep', e
+ logging.debug('arv-mount exception keep %s', e)
return False
def __getitem__(self, item):
- if item not in self._entries:
- self._entries[item] = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, item))
- return self._entries[item]
-
+ if item in self:
+ return self._entries[item]
+ else:
+ raise KeyError("No collection with id " + item)
class TagsDirectory(Directory):
'''A special directory that contains as subdirectories all tags visible to the user.'''
def update(self):
tags = self.api.links().list(filters=[['link_class', '=', 'tag']], select=['name'], distinct = True).execute()
- self.merge(tags['items'],
- lambda i: i['name'],
- lambda a, i: a.tag == i,
- lambda i: TagDirectory(self.inode, self.inodes, self.api, i['name'], poll=self._poll, poll_time=self._poll_time))
+ if "items" in tags:
+ self.merge(tags['items'],
+ lambda i: i['name'],
+ lambda a, i: a.tag == i,
+ lambda i: TagDirectory(self.inode, self.inodes, self.api, i['name'], poll=self._poll, poll_time=self._poll_time))
class TagDirectory(Directory):
'''A special directory that contains as subdirectories all collections visible
so request handlers do not run concurrently unless the lock is explicitly released
with llfuse.lock_released.'''
- def __init__(self, uid, gid):
+ def __init__(self, uid, gid, debug=False):
super(Operations, self).__init__()
+ if debug:
+ logging.basicConfig(level=logging.DEBUG)
+ logging.info("arv-mount debug enabled")
+
self.inodes = Inodes()
self.uid = uid
self.gid = gid
return entry
def lookup(self, parent_inode, name):
- #print "lookup: parent_inode", parent_inode, "name", name
+ logging.debug("arv-mount lookup: parent_inode %i name %s", parent_inode, name)
inode = None
if name == '.':
return fh
def read(self, fh, off, size):
- #print "read", fh, off, size
+ logging.debug("arv-mount read %i %i %i", fh, off, size)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
del self._filehandles[fh]
def opendir(self, inode):
- #print "opendir: inode", inode
+ logging.debug("arv-mount opendir: inode %i", inode)
if inode in self.inodes:
p = self.inodes[inode]
return fh
def readdir(self, fh, off):
- #print "readdir: fh", fh, "off", off
+ logging.debug("arv-mount readdir: fh %i off %i", fh, off)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
raise llfuse.FUSEError(errno.EBADF)
- #print "handle.entry", handle.entry
+ logging.debug("arv-mount handle.entry %s", handle.entry)
e = off
while e < len(handle.entry):
#!/usr/bin/env python
-from arvados_fuse import *
-import arvados
-import subprocess
import argparse
+import arvados
import daemon
+import os
import signal
+import subprocess
+
+from arvados_fuse import *
if __name__ == '__main__':
# Handle command line parameters
args = parser.parse_args()
# Create the request handler
- operations = Operations(os.getuid(), os.getgid())
+ operations = Operations(os.getuid(), os.getgid(), args.debug)
+
+ if args.debug:
+ arvados.config.settings()['ARVADOS_DEBUG'] = 'true'
if args.groups:
api = arvados.api('v1')
exit(rc)
else:
- if args.foreground:
- # Initialize the fuse connection
- llfuse.init(operations, args.mountpoint, opts)
- llfuse.main()
- else:
- # Initialize the fuse connection
- llfuse.init(operations, args.mountpoint, opts)
- with daemon.DaemonContext():
- llfuse.main()
+ os.chdir(args.mountpoint)
+ if not args.foreground:
+ daemon_ctx = daemon.DaemonContext(working_directory='.')
+ daemon_ctx.open()
+ llfuse.init(operations, '.', opts)
+ llfuse.main()
projects / arvados.git / commitdiff