preload_objects_for_dataclass(Container, recent_cr_containers) if recent_cr_containers.andand.any?
# fetch children of all the active crs in one call, if there are any
- active_crs = recent_crs.each {|cr| cr if (cr.priority > 0 and cr.state != 'Final' and cr.container_uuid)}
+ active_crs = recent_crs.each {|cr| cr if (cr.priority.andand > 0 and cr.state != 'Final' and cr.container_uuid)}
active_cr_uuids = active_crs.map(&:uuid)
active_cr_containers = active_crs.map {|cr| cr.container_uuid}.compact.uniq
cr_children = {}
<div>
<table id="trash-index" class="topalign table table-condensed table-fixedlayout">
<colgroup>
- <col width="5%" />
+ <col width="2%" />
<col width="20%" />
+ <col width="13%" />
<col width="15%" />
- <col width="15%" />
- <col width="10%" />
- <col width="30%" />
+ <col width="20%" />
+ <col width="25%" />
<col width="5%" />
</colgroup>
<tr class="contain-align-left">
<th></th>
<th>Name</th>
- <th>Trashed at</th>
- <th title="After this time, no longer available to be recovered from Trash">Permanently<br/>Deleted At</th>
+ <th>Date trashed /<br />to be deleted</th>
<th>Owner</th>
+ <th>UUID /<br />Content address (PDH)</th>
<th>Contents</th>
<th></th>
</tr>
</td>
<td>
<%= if !obj.name.blank? then obj.name else obj.uuid end %>
+ </td>
<td>
- <%= render_localized_date(obj.trash_at) if obj.trash_at %>
- <td>
- <%= render_localized_date(obj.delete_at) if obj.delete_at %>
+ <% if obj.trash_at %>
+ <%= render_localized_date(obj.trash_at) %>
+ <% end %>
+ <br />
+ <% if obj.delete_at %>
+ <%= render_localized_date(obj.delete_at) %>
+ <% end %>
</td>
<td>
<%= link_to_if_arvados_object obj.owner_uuid, friendly_name: true %>
</td>
+ <td>
+ <%= obj.uuid %><br /><%= obj.portable_data_hash %>
+ </td>
<td>
<% for i in (0..[2, obj.files.length-1].min) %>
<% file = obj.files[i] %>
visit page_with_token('active', "/trash")
assert_text deleted['name']
+ assert_text deleted['uuid']
+ assert_text deleted['portable_data_hash']
assert_text expired1['name']
assert_no_text expired2['name'] # not readable by this user
assert_no_text 'foo_file' # not trash
visit page_with_token('active', "/trash")
assert_text deleted['name']
+ assert_text deleted['uuid']
+ assert_text deleted['portable_data_hash']
assert_text expired['name']
page.find_field('Search trash').set 'expired'
- assert_text expired['name']
assert_no_text deleted['name']
+ assert_text expired['name']
+
+ page.find_field('Search trash').set deleted['portable_data_hash'][0..9]
+
+ assert_no_text expired['name']
+ assert_text deleted['name']
+ assert_text deleted['uuid']
+ assert_text deleted['portable_data_hash']
click_button 'Selection...'
within('.selection-action-container') do
COMMAND_ARR+=(--deb-ignore-iteration-in-dependencies)
fi
+ # 12271 - As FPM-generated packages don't include scripts by default, the
+ # packages cleanup on upgrade depends on files being listed on the %files
+ # section in the generated SPEC files. To remove DIRECTORIES, they need to
+ # be listed in that sectiontoo, so we need to add this parameter to properly
+ # remove lingering dirs.
+ if [[ rpm = "$FORMAT" ]]; then
+ COMMAND_ARR+=('--rpm-auto-add-directories')
+ fi
+
if [[ "${DEBUG:-0}" != "0" ]]; then
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
if not images:
# Fetch Docker image if necessary.
- cwltool.docker.get_image(dockerRequirement, pull_image)
+ try:
+ cwltool.docker.get_image(dockerRequirement, pull_image)
+ except OSError as e:
+ raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
# Upload image to Arvados
args = []
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
-crunchrunner_re = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.(tmpdir|outdir|keep)\)=(.*)")
+crunchrunner_re = re.compile(r"^.*crunchrunner: \$\(task\.(tmpdir|outdir|keep)\)=(.*)$")
crunchrunner_git_commit = 'a3f2cb186e437bfce0031b024b2157b73ed2717d'
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
log = logc.open(logc.keys()[0])
- dirs = {}
- tmpdir = None
- outdir = None
- keepdir = None
+ dirs = {
+ "tmpdir": "/tmpdir",
+ "outdir": "/outdir",
+ "keep": "/keep"
+ }
for l in log:
- # Determine the tmpdir, outdir and keepdir paths from
+ # Determine the tmpdir, outdir and keep paths from
# the job run. Unfortunately, we can't take the first
# values we find (which are expected to be near the
# top) and stop scanning because if the node fails and
arv-keepdocker --pull arvados/jobs $tag
else
jobsimg=\$(curl http://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")
- arv-keepdocker --pull arvados/jobs $jobsimg
- docker tag -f arvados/jobs:$jobsimg arvados/jobs:latest
+ arv-keepdocker --pull arvados/jobs \$jobsimg
+ docker tag arvados/jobs:\$jobsimg arvados/jobs:latest
arv-keepdocker arvados/jobs latest
fi
this.Hash.Write(p[:n])
}
if err == io.EOF {
- sum := this.Hash.Sum(make([]byte, 0, this.Hash.Size()))
+ sum := this.Hash.Sum(nil)
if fmt.Sprintf("%x", sum) != this.Check {
err = BadChecksum
}
return n, err
}
-// WriteTo writes the entire contents of this.Reader to dest. Returns
-// BadChecksum if the checksum doesn't match.
+// WriteTo writes the entire contents of this.Reader to dest. Returns
+// BadChecksum if writing is successful but the checksum doesn't
+// match.
func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
if writeto, ok := this.Reader.(io.WriterTo); ok {
written, err = writeto.WriteTo(io.MultiWriter(dest, this.Hash))
written, err = io.Copy(io.MultiWriter(dest, this.Hash), this.Reader)
}
- sum := this.Hash.Sum(make([]byte, 0, this.Hash.Size()))
+ if err != nil {
+ return written, err
+ }
+ sum := this.Hash.Sum(nil)
if fmt.Sprintf("%x", sum) != this.Check {
- err = BadChecksum
+ return written, BadChecksum
}
- return written, err
+ return written, nil
}
// Close reads all remaining data from the underlying Reader and
return err
}
- sum := this.Hash.Sum(make([]byte, 0, this.Hash.Size()))
+ sum := this.Hash.Sum(nil)
if fmt.Sprintf("%x", sum) != this.Check {
err = BadChecksum
}
"bytes"
"crypto/md5"
"fmt"
- . "gopkg.in/check.v1"
"io"
"io/ioutil"
+
+ . "gopkg.in/check.v1"
)
type HashcheckSuiteSuite struct{}
c.Check(err, Equals, BadChecksum)
<-done
}
+
+ // If WriteTo stops early due to a write error, return the
+ // write error (not "bad checksum").
+ {
+ input := bytes.NewBuffer(make([]byte, 1<<26))
+ hcr := HashCheckingReader{input, md5.New(), hash}
+ r, w := io.Pipe()
+ r.Close()
+ n, err := hcr.WriteTo(w)
+ c.Check(n, Equals, int64(0))
+ c.Check(err, NotNil)
+ c.Check(err, Not(Equals), BadChecksum)
+ }
}
end
# Given a repository (url, or name of hosted repo) and commit sha1,
- # copy the commit into the internal git repo and tag it with the
- # given tag (typically a job UUID).
+ # copy the commit into the internal git repo (if necessary), and tag
+ # it with the given tag (typically a job UUID).
#
# The repo can be a remote url, but in this case sha1 must already
# be present in our local cache for that repo: e.g., sha1 was just
raise ArgumentError.new "no local repository for #{repo_name}"
end
dst_gitdir = Rails.configuration.git_internal_dir
- must_pipe("echo #{sha1.shellescape}",
- "git --git-dir #{src_gitdir.shellescape} pack-objects -q --revs --stdout",
- "git --git-dir #{dst_gitdir.shellescape} unpack-objects -q")
- must_git(dst_gitdir,
- "tag --force #{tag.shellescape} #{sha1.shellescape}")
+
+ begin
+ commit_in_dst = must_git(dst_gitdir, "log -n1 --format=%H #{sha1.shellescape}^{commit}").strip
+ rescue GitError
+ commit_in_dst = false
+ end
+
+ tag_cmd = "tag --force #{tag.shellescape} #{sha1.shellescape}^{commit}"
+ if commit_in_dst == sha1
+ must_git(dst_gitdir, tag_cmd)
+ else
+ # git-fetch is faster than pack-objects|unpack-objects, but
+ # git-fetch can't fetch by sha1. So we first try to fetch a
+ # branch that has the desired commit, and if that fails (there
+ # is no such branch, or the branch we choose changes under us in
+ # race), we fall back to pack|unpack.
+ begin
+ branches = must_git(src_gitdir,
+ "branch --contains #{sha1.shellescape}")
+ m = branches.match(/^. (\w+)\n/)
+ if !m
+ raise GitError.new "commit is not on any branch"
+ end
+ branch = m[1]
+ must_git(dst_gitdir,
+ "fetch file://#{src_gitdir.shellescape} #{branch.shellescape}")
+ # Even if all of the above steps succeeded, we might still not
+ # have the right commit due to a race, in which case tag_cmd
+ # will fail, and we'll need to fall back to pack|unpack. So
+ # don't be tempted to condense this tag_cmd and the one in the
+ # rescue block into a single attempt.
+ must_git(dst_gitdir, tag_cmd)
+ rescue GitError
+ must_pipe("echo #{sha1.shellescape}",
+ "git --git-dir #{src_gitdir.shellescape} pack-objects -q --revs --stdout",
+ "git --git-dir #{dst_gitdir.shellescape} unpack-objects -q")
+ must_git(dst_gitdir, tag_cmd)
+ end
+ end
end
protected
# Clear token in case a git helper tries to use it as a password.
orig_token = ENV['ARVADOS_API_TOKEN']
ENV['ARVADOS_API_TOKEN'] = ''
+ last_output = ''
begin
git = "git --git-dir #{gitdir.shellescape}"
cmds.each do |cmd|
- must_pipe git+" "+cmd
+ last_output = must_pipe git+" "+cmd
end
ensure
ENV['ARVADOS_API_TOKEN'] = orig_token
end
+ return last_output
end
def self.must_pipe *cmds
if not $?.success?
raise GitError.new "#{cmd}: #{$?}: #{out}"
end
+ return out
end
end
when Running
permitted.push :finished_at, :output, :log
when Queued, Locked
- permitted.push :finished_at
+ permitted.push :finished_at, :log
end
else
i, o, e, t = Open3.popen3(*cmd_args)
rescue
$stderr.puts "dispatch: popen3: #{$!}"
- sleep 1
- next
+ # This is a dispatch problem like "Too many open files";
+ # retrying another job right away would be futile. Just return
+ # and hope things are better next time, after (at least) a
+ # did_recently() delay.
+ return
end
$stderr.puts "dispatch: job #{job.uuid}"
pid_done = nil
j_done = nil
- if false
- begin
- pid_done = waitpid(-1, Process::WNOHANG | Process::WUNTRACED)
- if pid_done
- j_done = @running.values.
- select { |j| j[:wait_thr].pid == pid_done }.
- first
- end
- rescue SystemCallError
- # I have @running processes but system reports I have no
- # children. This is likely to happen repeatedly if it happens at
- # all; I will log this no more than once per child process I
- # start.
- if 0 < @running.select { |uuid,j| j[:warned_waitpid_error].nil? }.size
- children = @running.values.collect { |j| j[:wait_thr].pid }.join ' '
- $stderr.puts "dispatch: IPC bug: waitpid() error (#{$!}), but I have children #{children}"
- end
- @running.each do |uuid,j| j[:warned_waitpid_error] = true end
- end
- else
- @running.each do |uuid, j|
- if j[:wait_thr].status == false
- pid_done = j[:wait_thr].pid
- j_done = j
- end
+ @running.each do |uuid, j|
+ if !j[:wait_thr].status
+ pid_done = j[:wait_thr].pid
+ j_done = j
+ break
end
end
system("tar", "-xC", @tmpdir.to_s, "-f", "test/test.git.tar")
Rails.configuration.git_repositories_dir = "#{@tmpdir}/test"
- intdir = Rails.configuration.git_internal_dir
- if not File.exist? intdir
- FileUtils.mkdir_p intdir
- IO.read("|git --git-dir #{intdir.to_s.shellescape} init")
- assert $?.success?
- end
+ # Initialize an empty internal git repo.
+ intdir =
+ Rails.configuration.git_internal_dir =
+ Rails.root.join(@tmpdir, 'internal.git').to_s
+ FileUtils.mkdir_p intdir
+ IO.read("|git --git-dir #{intdir.shellescape} init")
+ assert $?.success?
end
base.teardown do
end
end
+ def must_pipe(cmd)
+ begin
+ return IO.read("|#{cmd}")
+ ensure
+ assert $?.success?
+ end
+ end
+
[
'https://github.com/curoverse/arvados.git',
'http://github.com/curoverse/arvados.git',
assert $?.success?
end
+ def with_foo_repository
+ Dir.chdir("#{Rails.configuration.git_repositories_dir}/#{repositories(:foo).uuid}") do
+ must_pipe("git checkout master 2>&1")
+ yield
+ end
+ end
+
+ test 'tag_in_internal_repository, new non-tip sha1 in local repo' do
+ tag = "tag#{rand(10**10)}"
+ sha1 = nil
+ with_foo_repository do
+ must_pipe("git checkout -b branch-#{rand(10**10)} 2>&1")
+ must_pipe("echo -n #{tag.shellescape} >bar")
+ must_pipe("git add bar")
+ must_pipe("git -c user.email=x@x -c user.name=X commit -m -")
+ sha1 = must_pipe("git log -n1 --format=%H").strip
+ must_pipe("git rm bar")
+ must_pipe("git -c user.email=x@x -c user.name=X commit -m -")
+ end
+ Commit.tag_in_internal_repository 'active/foo', sha1, tag
+ gitint = "git --git-dir #{Rails.configuration.git_internal_dir.shellescape}"
+ assert_match(/^commit /, IO.read("|#{gitint} show #{tag.shellescape}"))
+ assert $?.success?
+ end
+
+ test 'tag_in_internal_repository, new unreferenced sha1 in local repo' do
+ tag = "tag#{rand(10**10)}"
+ sha1 = nil
+ with_foo_repository do
+ must_pipe("echo -n #{tag.shellescape} >bar")
+ must_pipe("git add bar")
+ must_pipe("git -c user.email=x@x -c user.name=X commit -m -")
+ sha1 = must_pipe("git log -n1 --format=%H").strip
+ must_pipe("git reset --hard HEAD^")
+ end
+ Commit.tag_in_internal_repository 'active/foo', sha1, tag
+ gitint = "git --git-dir #{Rails.configuration.git_internal_dir.shellescape}"
+ assert_match(/^commit /, IO.read("|#{gitint} show #{tag.shellescape}"))
+ assert $?.success?
+ end
+
# In active/shabranchnames, "7387838c69a21827834586cc42b467ff6c63293b" is
# both a commit hash, and the name of a branch that begins from that same
# commit.
check_no_change_from_cancelled c
end
+ test "Container locked cancel with log" do
+ c, _ = minimal_new
+ set_user_from_auth :dispatch1
+ assert c.lock, show_errors(c)
+ assert c.update_attributes(
+ state: Container::Cancelled,
+ log: collections(:real_log_collection).portable_data_hash,
+ ), show_errors(c)
+ check_no_change_from_cancelled c
+ end
+
test "Container running cancel" do
c, _ = minimal_new
set_user_from_auth :dispatch1
if obj.in_use():
_logger.debug("InodeCache cannot clear inode %i, in use", obj.inode)
return
+ obj.kernel_invalidate()
if obj.has_ref(True):
- obj.kernel_invalidate()
_logger.debug("InodeCache sent kernel invalidate inode %i", obj.inode)
return
obj.clear()
del self._entries[entry.inode]
with llfuse.lock_released:
entry.finalize()
- self.invalidate_inode(entry.inode)
entry.inode = None
else:
entry.dead = True
_logger.debug("del_entry on inode %i with refcount %i", entry.inode, entry.ref_count)
- def invalidate_inode(self, inode):
- llfuse.invalidate_inode(inode)
+ def invalidate_inode(self, entry):
+ if entry.has_ref(False):
+ # Only necessary if the kernel has previously done a lookup on this
+ # inode and hasn't yet forgotten about it.
+ llfuse.invalidate_inode(entry.inode)
- def invalidate_entry(self, inode, name):
- llfuse.invalidate_entry(inode, name.encode(self.encoding))
+ def invalidate_entry(self, entry, name):
+ if entry.has_ref(False):
+ # Only necessary if the kernel has previously done a lookup on this
+ # inode and hasn't yet forgotten about it.
+ llfuse.invalidate_entry(entry.inode, name.encode(self.encoding))
def clear(self):
self.inode_cache.clear()
entry = llfuse.EntryAttributes()
entry.st_ino = inode
entry.generation = 0
- entry.entry_timeout = 60 if e.allow_dirent_cache else 0
- entry.attr_timeout = 60 if e.allow_attr_cache else 0
+ entry.entry_timeout = 0
+ entry.attr_timeout = e.time_to_next_poll() if e.allow_attr_cache else 0
entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
if isinstance(e, Directory):
self.dead = False
self.cache_size = 0
self.cache_uuid = None
+
+ # Can the kernel cache attributes?
self.allow_attr_cache = True
- self.allow_dirent_cache = True
def invalidate(self):
"""Indicate that object contents should be refreshed from source."""
def child_event(self, ev):
pass
+
+ def time_to_next_poll(self):
+ if self._poll:
+ t = (self._last_update + self._poll_time) - self._atime
+ if t < 0:
+ return 0
+ else:
+ return t
+ else:
+ return self._poll_time
# delete any other directory entries that were not in found in 'items'
for i in oldentries:
_logger.debug("Forgetting about entry '%s' on inode %i", i, self.inode)
- self.inodes.invalidate_entry(self.inode, i.encode(self.inodes.encoding))
+ self.inodes.invalidate_entry(self, i)
self.inodes.del_entry(oldentries[i])
changed = True
if changed:
- self.inodes.invalidate_inode(self.inode)
+ self.inodes.invalidate_inode(self)
self._mtime = time.time()
self.fresh()
self._entries = {}
for n in oldentries:
oldentries[n].clear()
- self.inodes.invalidate_entry(self.inode, n.encode(self.inodes.encoding))
self.inodes.del_entry(oldentries[n])
- self.inodes.invalidate_inode(self.inode)
self.invalidate()
def kernel_invalidate(self):
- for n, e in self._entries.iteritems():
- self.inodes.invalidate_entry(self.inode, n.encode(self.inodes.encoding))
- e.kernel_invalidate()
- self.inodes.invalidate_inode(self.inode)
+ # Invalidating the dentry on the parent implies invalidating all paths
+ # below it as well.
+ parent = self.inodes[self.parent_inode]
+
+ # Find self on the parent in order to invalidate this path.
+ # Calling the public items() method might trigger a refresh,
+ # which we definitely don't want, so read the internal dict directly.
+ for k,v in parent._entries.items():
+ if v is self:
+ self.inodes.invalidate_entry(parent, k)
+ break
def mtime(self):
return self._mtime
elif event == arvados.collection.DEL:
ent = self._entries[name]
del self._entries[name]
- self.inodes.invalidate_entry(self.inode, name.encode(self.inodes.encoding))
+ self.inodes.invalidate_entry(self, name)
self.inodes.del_entry(ent)
elif event == arvados.collection.MOD:
if hasattr(item, "fuse_entry") and item.fuse_entry is not None:
- self.inodes.invalidate_inode(item.fuse_entry.inode)
+ self.inodes.invalidate_inode(item.fuse_entry)
elif name in self._entries:
- self.inodes.invalidate_inode(self._entries[name].inode)
+ self.inodes.invalidate_inode(self._entries[name])
def populate(self, mtime):
self._mtime = mtime
if self.collection_record_file:
with llfuse.lock:
self.collection_record_file.invalidate()
- self.inodes.invalidate_inode(self.collection_record_file.inode)
+ self.inodes.invalidate_inode(self.collection_record_file)
_logger.debug("%s invalidated collection record", self)
def collection_record(self):
return False
try:
+ e = None
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
self.inodes.del_entry(e)
return True
else:
- self.inodes.invalidate_entry(self.inode, k)
+ self.inodes.invalidate_entry(self, k)
self.inodes.del_entry(e)
return False
except Exception as ex:
- _logger.debug('arv-mount exception keep %s', ex)
- self.inodes.del_entry(e)
+ _logger.exception("arv-mount lookup '%s':", k)
+ if e is not None:
+ self.inodes.del_entry(e)
return False
def __getitem__(self, item):
# Acually move the entry from source directory to this directory.
del src._entries[name_old]
self._entries[name_new] = ent
- self.inodes.invalidate_entry(src.inode, name_old.encode(self.inodes.encoding))
+ self.inodes.invalidate_entry(src, name_old)
@use_counter
def child_event(self, ev):
if old_name in self._entries:
ent = self._entries[old_name]
del self._entries[old_name]
- self.inodes.invalidate_entry(self.inode, old_name.encode(self.inodes.encoding))
+ self.inodes.invalidate_entry(self, old_name)
if new_name:
if ent is not None:
super(FuncToJSONFile, self).__init__(parent_inode, "", 0)
self.func = func
- # invalidate_inode() and invalidate_entry() are asynchronous
- # with no callback to wait for. In order to guarantee
- # userspace programs don't get stale data that was generated
- # before the last invalidate(), we must disallow dirent
+ # invalidate_inode() is asynchronous with no callback to wait for. In
+ # order to guarantee userspace programs don't get stale data that was
+ # generated before the last invalidate(), we must disallow inode
# caching entirely.
- self.allow_dirent_cache = False
+ self.allow_attr_cache = False
def size(self):
self._update()
PASSENGER="$ARVBOX_DATA/passenger"
GEMS="$ARVBOX_DATA/gems"
PIPCACHE="$ARVBOX_DATA/pip"
+NPMCACHE="$ARVBOX_DATA/npm"
GOSTUFF="$ARVBOX_DATA/gopath"
getip() {
updateconf
wait_for_arvbox
else
- mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$GOSTUFF"
+ mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$NPMCACHE" "$GOSTUFF"
if ! test -d "$ARVADOS_ROOT" ; then
"--volume=$PASSENGER:/var/lib/passenger:rw" \
"--volume=$GEMS:/var/lib/gems:rw" \
"--volume=$PIPCACHE:/var/lib/pip:rw" \
+ "--volume=$NPMCACHE:/var/lib/npm:rw" \
"--volume=$GOSTUFF:/var/lib/gopath:rw" \
arvados/arvbox-dev$TAG \
/usr/local/bin/runsvinit -svdir=/etc/test-service
"--volume=$PASSENGER:/var/lib/passenger:rw" \
"--volume=$GEMS:/var/lib/gems:rw" \
"--volume=$PIPCACHE:/var/lib/pip:rw" \
+ "--volume=$NPMCACHE:/var/lib/npm:rw" \
"--volume=$GOSTUFF:/var/lib/gopath:rw" \
$PUBLIC \
arvados/arvbox-dev$TAG
GOPATH=$PWD go get github.com/curoverse/runsvinit && \
install bin/runsvinit /usr/local/bin
-ENV PJSVERSION=1.9.7
+ENV PJSVERSION=1.9.8
+# bitbucket is the origin, but downloads fail sometimes, so use our own mirror instead.
+#ENV PJSURL=https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2
+ENV PJSURL=http://cache.arvados.org/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2
RUN set -e && \
- curl -L -f http://cache.arvados.org/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2 | tar -C /usr/local -xjf - && \
+ curl -L -f ${PJSURL} | tar -C /usr/local -xjf - && \
ln -s ../phantomjs-${PJSVERSION}-linux-x86_64/bin/phantomjs /usr/local/bin
RUN pip install -U setuptools
RUN curl -L -f https://nodejs.org/dist/${NODEVERSION}/node-${NODEVERSION}-linux-x64.tar.xz | tar -C /usr/local -xJf - && \
ln -s ../node-${NODEVERSION}-linux-x64/bin/node ../node-${NODEVERSION}-linux-x64/bin/npm /usr/local/bin
+RUN echo en_US.UTF-8 UTF-8 > /etc/locale.gen && locale-gen
+
ARG arvados_version
RUN echo arvados_version is git commit $arvados_version
export PATH=${PATH}:/usr/local/go/bin:/var/lib/gems/bin
export GEM_HOME=/var/lib/gems
export GEM_PATH=/var/lib/gems
+export npm_config_cache=/var/lib/npm
+export npm_config_cache_min=Infinity
if test -s /var/run/localip_override ; then
localip=$(cat /var/run/localip_override)
done
popd
- if ! pip install --no-index --find-links /var/lib/pip --system $1 ; then
- pip install --system $1
+ if ! pip install --no-index --find-links /var/lib/pip $1 ; then
+ pip install $1
fi
}
HOSTGID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f5)
mkdir -p /var/lib/arvados/git /var/lib/gems \
- /var/lib/passenger /var/lib/gopath /var/lib/pip
+ /var/lib/passenger /var/lib/gopath \
+ /var/lib/pip /var/lib/npm
groupadd --gid $HOSTGID --non-unique arvbox
groupadd --gid $HOSTGID --non-unique git
chown arvbox:arvbox -R /usr/local /var/lib/arvados /var/lib/gems \
/var/lib/passenger /var/lib/postgresql \
/var/lib/nginx /var/log/nginx /etc/ssl/private \
- /var/lib/gopath /var/lib/pip
+ /var/lib/gopath /var/lib/pip /var/lib/npm
mkdir -p /var/lib/gems/ruby
chown arvbox:arvbox -R /var/lib/gems/ruby
PGVERSION=9.6
if ! test -d /var/lib/postgresql/$PGVERSION/main ; then
- /usr/lib/postgresql/$PGVERSION/bin/initdb -D /var/lib/postgresql/$PGVERSION/main
+ /usr/lib/postgresql/$PGVERSION/bin/initdb --locale=en_US.UTF-8 -D /var/lib/postgresql/$PGVERSION/main
sh -c "while ! (psql postgres -c'\du' | grep '^ arvbox ') >/dev/null ; do createuser -s arvbox ; sleep 1 ; done" &
fi
mkdir -p /var/run/postgresql/$PGVERSION-main.pg_stat_tmp
run_bundler --binstubs=$PWD/binstubs
ln -sf /usr/src/arvados/sdk/cli/binstubs/arv /usr/local/bin/arv
+# Need to install the upstream version of pip because the python-pip package
+# shipped with Debian 9 is patched to change behavior in a way that breaks our
+# use case.
+# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=876145
+# When a non-root user attempts to install system packages, it makes the
+# --ignore-installed flag the default (and there is no way to turn it off),
+# this has the effect of making it very hard to share dependencies shared among
+# multiple packages, because it will blindly install the latest version of each
+# dependency requested by each package, even if a compatible package version is
+# already installed.
+pip_install pip
+
pip_install wheel
cd /usr/src/arvados/sdk/python
projects / arvados.git / commitdiff