IO.popen(['arv-normalize'], 'w+b') do |io|
io.write combined
io.close_write
- while buf = io.read(2**20)
+ while buf = io.read(2**16)
normalized += buf
end
end
+ normalized_stripped = ''
+ IO.popen(['arv-normalize', '--strip'], 'w+b') do |io|
+ io.write combined
+ io.close_write
+ while buf = io.read(2**16)
+ normalized_stripped += buf
+ end
+ end
+
require 'digest/md5'
d = Digest::MD5.new()
- d << normalized
+ d << normalized_stripped
newuuid = "#{d.hexdigest}+#{normalized.length}"
env = Hash[ENV].
def get_n_objects_of_class dataclass, size
@objects_map_for ||= {}
- raise ArgumentError, 'Argument is not a data class' unless dataclass.is_a? Class
+ raise ArgumentError, 'Argument is not a data class' unless dataclass.is_a? ArvadosBase
raise ArgumentError, 'Argument is not a valid limit size' unless (size && size>0)
# if the objects_map_for has a value for this dataclass, and the
selectables = []
attrtext = attrvalue
- if dataclass and dataclass.is_a? Class
+ if dataclass and dataclass.is_a? ArvadosBase
objects = get_n_objects_of_class dataclass, 10
objects.each do |item|
items << item
render opts.merge(partial: "application/#{partial}")
end
end
-
+
def fa_icon_class_for_object object
case object.class.to_s.to_sym
when :User
:repository => c[:repository],
:nondeterministic => c[:nondeterministic],
:output_is_persistent => c[:output_is_persistent] || false,
+ :runtime_constraints => c[:runtime_constraints],
:owner_uuid => owner_uuid,
# TODO: Delete the following three attributes when
# supporting pre-20140418 API servers is no longer
ended += 1
if c[:job][:success] == true
succeeded += 1
- elsif c[:job][:success] == false
+ elsif c[:job][:success] == false or c[:job][:cancelled_at]
failed += 1
end
end
my $command =
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
."mkdir -p $ENV{JOB_WORK} $ENV{CRUNCH_TMP} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
+ ."&& chmod og+wrx $ENV{TASK_WORK}"
."&& cd $ENV{CRUNCH_TMP} ";
+ umask(077);
if ($build_script)
{
$build_script_to_send = $build_script;
$command .=
q{$(ip -o address show scope global |
gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
- foreach my $env_key (qw(CRUNCH_SRC CRUNCH_TMP TASK_KEEPMOUNT))
- {
- $command .= "-v \Q$ENV{$env_key}:$ENV{$env_key}:rw\E ";
- }
+ $command .= "-v \Q$ENV{TASK_WORK}:/tmp/crunch-job:rw\E ";
+ $command .= "-v \Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
+ $command .= "-v \Q$ENV{TASK_KEEPMOUNT}:/mnt:ro\E ";
while (my ($env_key, $env_val) = each %ENV)
{
if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
- $command .= "-e \Q$env_key=$env_val\E ";
+ if ($env_key eq "TASK_WORK") {
+ $command .= "-e \QTASK_WORK=/tmp/crunch-job\E ";
+ }
+ elsif ($env_key eq "TASK_KEEPMOUNT") {
+ $command .= "-e \QTASK_KEEPMOUNT=/mnt\E ";
+ }
+ elsif ($env_key eq "CRUNCH_SRC") {
+ $command .= "-e \QCRUNCH_SRC=/tmp/crunch-src\E ";
+ }
+ else {
+ $command .= "-e \Q$env_key=$env_val\E ";
+ }
}
}
$command .= "\Q$docker_hash\E ";
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 "
}
$command .= "stdbuf -o0 -e0 ";
- $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+ $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
exit (111);
use Arvados;
$arv = Arvados->new(apiHost => 'arvados.local');
-
+
my $instances = $arv->{'pipeline_instances'}->{'list'}->execute();
print "UUID is ", $instances->{'items'}->[0]->{'uuid'}, "\n";
-
+
$uuid = 'eiv0u-arx5y-2c5ovx43zw90gvh';
$instance = $arv->{'pipeline_instances'}->{'get'}->execute('uuid' => $uuid);
print "ETag is ", $instance->{'etag'}, "\n";
-
+
$instance->{'active'} = 1;
$instance->{'name'} = '';
$instance->save();
=cut
package Arvados;
+
+use Net::SSL (); # From Crypt-SSLeay
+BEGIN {
+ $Net::HTTPS::SSL_SOCKET_CLASS = "Net::SSL"; # Force use of Net::SSL
+}
+
use JSON;
-use Data::Dumper;
-use IO::Socket::SSL;
use Carp;
use Arvados::ResourceAccessor;
use Arvados::ResourceMethod;
use Arvados::ResourceProxy;
use Arvados::ResourceProxyList;
use Arvados::Request;
+use Data::Dumper;
$Arvados::VERSION = 0.1;
$config = load_config_file("$ENV{HOME}/.config/arvados/settings.conf");
- $self->{'authToken'} ||=
+ $self->{'authToken'} ||=
$ENV{ARVADOS_API_TOKEN} || $config->{ARVADOS_API_TOKEN};
$self->{'apiHost'} ||=
$ENV{ARVADOS_API_HOST} || $config->{ARVADOS_API_HOST};
+ $self->{'noVerifyHostname'} ||=
+ $ENV{ARVADOS_API_HOST_INSECURE};
+
$self->{'apiProtocolScheme'} ||=
$ENV{ARVADOS_API_PROTOCOL_SCHEME} ||
$config->{ARVADOS_API_PROTOCOL_SCHEME};
{
my $self = shift;
local $ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'};
- if ($opts{'noVerifyHostname'} || ($host =~ /\.local$/)) {
+ if ($self{'noVerifyHostname'} || ($host =~ /\.local$/)) {
$ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0;
}
Arvados::Request->new();
for f in s.all_files():
yield f
- def manifest_text(self):
+ def manifest_text(self, strip=False):
self._populate()
- return self._manifest_text
+ if strip:
+ m = ''.join([StreamReader(stream).manifest_text(strip=True) for stream in self._streams])
+ return m
+ else:
+ return self._manifest_text
class CollectionWriter(object):
KEEP_BLOCK_SIZE = 2**26
for x in fields[1:-1] ]
clean += fields[0] + ' ' + ' '.join(locators) + ' ' + fields[-1] + "\n"
return clean
-
+
def manifest_text(self):
self.finish_current_stream()
manifest = ''
block_size = data_locators[i][BLOCKSIZE]
block_start = data_locators[i][OFFSET]
block_end = block_start + block_size
-
+
while i < len(data_locators):
locator, block_size, block_start = data_locators[i]
block_end = block_start + block_size
dc = bz2.BZ2Decompressor()
return self.decompress(lambda segment: dc.decompress(segment), size)
elif re.search('\.gz$', self._name):
- dc = zlib.decompressobj(16+zlib.MAX_WBITS)
+ dc = zlib.decompressobj(16+zlib.MAX_WBITS)
return self.decompress(lambda segment: dc.decompress(dc.unconsumed_tail + segment), size)
else:
return self.readall(size)
self._keep = keep
else:
self._keep = Keep.global_client_object()
-
+
streamoffset = 0L
# parse stream
for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._data_locators, start, size):
data += self._keep.get(locator)[segmentoffset:segmentoffset+segmentsize]
return data
-
- def manifest_text(self):
+
+ def manifest_text(self, strip=False):
manifest_text = [self.name().replace(' ', '\\040')]
- manifest_text.extend([d[LOCATOR] for d in self._data_locators])
- manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
+ if strip:
+ for d in self._data_locators:
+ m = re.match(r'^[0-9a-f]{32}\+(\d+)*', d[LOCATOR])
+ manifest_text.append(m.group(0))
+ else:
+ manifest_text.extend([d[LOCATOR] for d in self._data_locators])
+ manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
for seg in f.segments])
for f in self._files.values()])
return ' '.join(manifest_text) + '\n'
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', False)
p = subprocess.Popen(execargs, **kwargs)
- stdoutdata, stderrdata = p.communicate(None)
+ if kwargs['stdout'] == subprocess.PIPE:
+ stdoutdata, stderrdata = p.communicate(None)
+ else:
+ p.wait()
if p.returncode != 0:
raise errors.CommandFailedError(
"run_command %s exit %d:\n%s" %
break
zip_file.write(buf)
zip_file.close()
-
+
p = subprocess.Popen(["unzip",
"-q", "-o",
"-d", path,
description='Read manifest on standard input and put normalized manifest on standard output.')
parser.add_argument('--extract', type=str, help="The file to extract from the input manifest")
+parser.add_argument('--strip', action='store_true', help="Strip authorization tokens")
args = parser.parse_args()
import arvados
r = sys.stdin.read()
-
+
cr = arvados.CollectionReader(r)
if args.extract:
if fn in s.files():
sys.stdout.write(s.files()[fn].as_manifest())
else:
- sys.stdout.write(cr.manifest_text())
+ sys.stdout.write(cr.manifest_text(args.strip))
#!/usr/bin/env ruby
+
if ENV["CRUNCH_DISPATCH_LOCKFILE"]
lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
lockfile = File.open(lockfilename, File::RDWR|File::CREAT, 0644)
require File.dirname(__FILE__) + '/../config/boot'
require File.dirname(__FILE__) + '/../config/environment'
-def cancel_stale_jobs
- Job.running.each do |jobrecord|
- f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
- if f
- age = (Time.now - f.created_at)
- if age > 300
- $stderr.puts "dispatch: failing orphan job #{jobrecord.uuid}, last log is #{age} seconds old"
- # job is marked running, but not known to crunch-dispatcher, and
- # hasn't produced any log entries for 5 minutes, so mark it as failed.
- jobrecord.running = false
- jobrecord.cancelled_at ||= Time.now
- jobrecord.finished_at ||= Time.now
- if jobrecord.success.nil?
- jobrecord.success = false
+class CancelJobs
+ include ApplicationHelper
+
+ def cancel_stale_jobs
+ act_as_system_user do
+ Job.running.each do |jobrecord|
+ puts jobrecord
+ f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
+ if f
+ age = (Time.now - f.created_at)
+ if age > 300
+ $stderr.puts "dispatch: failing orphan job #{jobrecord.uuid}, last log is #{age} seconds old"
+ # job is marked running, but not known to crunch-dispatcher, and
+ # hasn't produced any log entries for 5 minutes, so mark it as failed.
+ jobrecord.running = false
+ jobrecord.cancelled_at ||= Time.now
+ jobrecord.finished_at ||= Time.now
+ if jobrecord.success.nil?
+ jobrecord.success = false
+ end
+ jobrecord.save!
+ end
end
- jobrecord.save!
end
end
end
end
-cancel_stale_jobs
+CancelJobs.new.cancel_stale_jobs
# FUSE driver for Arvados Keep
#
+from __future__ import print_function
import os
import sys
from time import time
from llfuse import FUSEError
+DEBUG = False
+
class FreshBase(object):
'''Base class for maintaining fresh/stale state to determine when to update.'''
def __init__(self):
try:
self.update()
except apiclient.errors.HttpError as e:
- print e
+ if DEBUG:
+ print(e, file=sys.stderr)
def __getitem__(self, item):
self.checkupdate()
cwd = cwd._entries[part]
for k, v in s.files().items():
cwd._entries[k] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v))
- print "found"
self.fresh()
+ return True
except Exception as detail:
- print("%s: error: %s" % (self.collection_locator,detail) )
+ if DEBUG:
+ print("arv-mount %s: error: %s" % (self.collection_locator,detail), file=sys.stderr)
+ return False
class MagicDirectory(Directory):
'''A special directory that logically contains the set of all extant keep
if k in self._entries:
return True
try:
- if arvados.Keep.get(k):
+ e = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, k))
+ if e.update():
+ self._entries[k] = e
return True
else:
return False
except Exception as e:
- #print 'exception keep', e
+ if DEBUG:
+ print('arv-mount exception keep', e, file=sys.stderr)
return False
def __getitem__(self, item):
- if item not in self._entries:
- self._entries[item] = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, item))
- return self._entries[item]
-
+ if self.__contains__(item):
+ return self._entries[item]
+ else:
+ raise KeyError()
class TagsDirectory(Directory):
'''A special directory that contains as subdirectories all tags visible to the user.'''
so request handlers do not run concurrently unless the lock is explicitly released
with llfuse.lock_released.'''
- def __init__(self, uid, gid):
+ def __init__(self, uid, gid, debug):
super(Operations, self).__init__()
+ if debug:
+ global DEBUG
+ DEBUG = True
+ print("arv-mount debug enabled", file=sys.stderr)
+
self.inodes = Inodes()
self.uid = uid
self.gid = gid
return entry
def lookup(self, parent_inode, name):
- #print "lookup: parent_inode", parent_inode, "name", name
+ if DEBUG:
+ print("arv-mount lookup: parent_inode", parent_inode, "name", name, file=sys.stderr)
inode = None
if name == '.':
return fh
def read(self, fh, off, size):
- #print "read", fh, off, size
+ if DEBUG:
+ print("arv-mount read", fh, off, size, file=sys.stderr)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
del self._filehandles[fh]
def opendir(self, inode):
- #print "opendir: inode", inode
+ if DEBUG:
+ print("arv-mount opendir: inode", inode, file=sys.stderr)
if inode in self.inodes:
p = self.inodes[inode]
return fh
def readdir(self, fh, off):
- #print "readdir: fh", fh, "off", off
+ if DEBUG:
+ print("arv-mount readdir: fh", fh, "off", off, file=sys.stderr)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
raise llfuse.FUSEError(errno.EBADF)
- #print "handle.entry", handle.entry
+ if DEBUG:
+ print("arv-mount handle.entry", handle.entry, file=sys.stderr)
e = off
while e < len(handle.entry):
args = parser.parse_args()
# Create the request handler
- operations = Operations(os.getuid(), os.getgid())
+ operations = Operations(os.getuid(), os.getgid(), args.debug)
+
+ if args.debug:
+ arvados.config.settings()['ARVADOS_DEBUG'] = 'true'
if args.groups:
api = arvados.api('v1')