$ arvbox
Arvados-in-a-box http://arvados.org
-build <config> build arvbox Docker image
-rebuild <config> build arvbox Docker image, no layer cache
-start|run <config> start arvbox container
-open open arvbox workbench in a web browser
-shell enter arvbox shell
-ip print arvbox docker container ip address
-host print arvbox published host
-status print some information about current arvbox
+start|run <config> [tag] start arvbox container
stop stop arvbox container
restart <config> stop, then run again
-reboot <config> stop, build arvbox Docker image, run
+status print some information about current arvbox
+ip print arvbox docker container ip address
+host print arvbox published host
+shell enter arvbox shell
+open open arvbox workbench in a web browser
+root-cert get copy of root certificate
+update <config> stop, pull latest image, run
+build <config> build arvbox Docker image
+reboot <config> stop, build arvbox Docker image, run
+rebuild <config> build arvbox Docker image, no layer cache
reset delete arvbox arvados data (be careful!)
destroy delete all arvbox code and data (be careful!)
log <service> tail log of specified service
clone <from> <to> clone an arvbox
</pre>
+h2. Install root certificate
+
+Arvbox creates root certificate to authorize Arvbox services. Installing the root certificate into your web browser will prevent security errors when accessing Arvbox services with your web browser. Every Arvbox instance generates a new root signing key.
+
+# Export the certificate using @arvbox root-cert@
+# Go to the certificate manager in your browser.
+#* In Chrome, this can be found under "Settings → Advanced → Manage Certificates" or by entering @chrome://settings/certificates@ in the URL bar.
+#* In Firefox, this can be found under "Preferences → Privacy & Security" or entering @about:preferences#privacy@ in the URL bar and then choosing "View Certificates...".
+# Select the "Authorities" tab, then press the "Import" button. Choose @arvbox-root-cert.pem@
+
+The certificate will be added under the "Arvados testing" organization as "arvbox testing root CA".
+
+To access your Arvbox instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage (instructions for Debian/Ubuntu):
+
+# copy @arvbox-root-cert.pem@ to @/usr/local/share/ca-certificates/@
+# run @/usr/sbin/update-ca-certificates@
+
h2. Configs
h3. dev
}
func (ai *azureInstance) Address() string {
- if ai.nic.IPConfigurations != nil &&
- len(*ai.nic.IPConfigurations) > 0 &&
- (*ai.nic.IPConfigurations)[0].InterfaceIPConfigurationPropertiesFormat != nil &&
- (*ai.nic.IPConfigurations)[0].InterfaceIPConfigurationPropertiesFormat.PrivateIPAddress != nil {
-
- return *(*ai.nic.IPConfigurations)[0].PrivateIPAddress
+ if iprops := ai.nic.InterfacePropertiesFormat; iprops == nil {
+ return ""
+ } else if ipconfs := iprops.IPConfigurations; ipconfs == nil || len(*ipconfs) == 0 {
+ return ""
+ } else if ipconfprops := (*ipconfs)[0].InterfaceIPConfigurationPropertiesFormat; ipconfprops == nil {
+ return ""
+ } else if addr := ipconfprops.PrivateIPAddress; addr == nil {
+ return ""
+ } else {
+ return *addr
}
- return ""
}
func (ai *azureInstance) RemoteUser() string {
return nil, fmt.Errorf("no host in config Services.Controller.ExternalURL: %v", ctrlURL)
}
return &Client{
- APIHost: fmt.Sprintf("%v", ctrlURL),
+ APIHost: ctrlURL.Host,
Insecure: cluster.TLS.Insecure,
}, nil
}
}
arv.Retries = 25
+ ctx, cancel := context.WithCancel(context.Background())
+
dispatcher := dispatch.Dispatcher{
Logger: logger,
Arv: arv,
- RunContainer: run,
+ RunContainer: (&LocalRun{startFunc, make(chan bool, 8), ctx}).run,
PollPeriod: time.Duration(*pollInterval) * time.Second,
}
- ctx, cancel := context.WithCancel(context.Background())
err = dispatcher.Run(ctx)
if err != nil {
return err
return cmd.Start()
}
-var startCmd = startFunc
+type LocalRun struct {
+ startCmd func(container arvados.Container, cmd *exec.Cmd) error
+ concurrencyLimit chan bool
+ ctx context.Context
+}
// Run a container.
//
//
// If the container is in any other state, or is not Complete/Cancelled after
// crunch-run terminates, mark the container as Cancelled.
-func run(dispatcher *dispatch.Dispatcher,
+func (lr *LocalRun) run(dispatcher *dispatch.Dispatcher,
container arvados.Container,
status <-chan arvados.Container) {
uuid := container.UUID
if container.State == dispatch.Locked {
+
+ select {
+ case lr.concurrencyLimit <- true:
+ break
+ case <-lr.ctx.Done():
+ return
+ }
+
+ defer func() { <-lr.concurrencyLimit }()
+
+ select {
+ case c := <-status:
+ // Check for state updates after possibly
+ // waiting to be ready-to-run
+ if c.Priority == 0 {
+ goto Finish
+ }
+ default:
+ break
+ }
+
waitGroup.Add(1)
+ defer waitGroup.Done()
cmd := exec.Command(*crunchRunCommand, uuid)
cmd.Stdin = nil
// succeed in starting crunch-run.
runningCmdsMutex.Lock()
- if err := startCmd(container, cmd); err != nil {
+ if err := lr.startCmd(container, cmd); err != nil {
runningCmdsMutex.Unlock()
dispatcher.Logger.Warnf("error starting %q for %s: %s", *crunchRunCommand, uuid, err)
dispatcher.UpdateState(uuid, dispatch.Cancelled)
delete(runningCmds, uuid)
runningCmdsMutex.Unlock()
}
- waitGroup.Done()
}
+Finish:
+
// If the container is not finalized, then change it to "Cancelled".
err := dispatcher.Arv.Get("containers", uuid, nil, &container)
if err != nil {
dispatcher := dispatch.Dispatcher{
Arv: arv,
PollPeriod: time.Second,
- RunContainer: func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
- run(d, c, s)
- cancel()
- },
}
- startCmd = func(container arvados.Container, cmd *exec.Cmd) error {
+ startCmd := func(container arvados.Container, cmd *exec.Cmd) error {
dispatcher.UpdateState(container.UUID, "Running")
dispatcher.UpdateState(container.UUID, "Complete")
return cmd.Start()
}
+ dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
+ (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ cancel()
+ }
+
err = dispatcher.Run(ctx)
c.Assert(err, Equals, context.Canceled)
dispatcher := dispatch.Dispatcher{
Arv: arv,
PollPeriod: time.Second / 20,
- RunContainer: func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
- run(d, c, s)
- cancel()
- },
}
- startCmd = func(container arvados.Container, cmd *exec.Cmd) error {
+ startCmd := func(container arvados.Container, cmd *exec.Cmd) error {
dispatcher.UpdateState(container.UUID, "Running")
dispatcher.UpdateState(container.UUID, "Complete")
return cmd.Start()
}
+ dispatcher.RunContainer = func(d *dispatch.Dispatcher, c arvados.Container, s <-chan arvados.Container) {
+ (&LocalRun{startCmd, make(chan bool, 8), ctx}).run(d, c, s)
+ cancel()
+ }
+
re := regexp.MustCompile(`(?ms).*` + expected + `.*`)
go func() {
for i := 0; i < 80 && !re.MatchString(buf.String()); i++ {
if locatorIn == "" {
bytes, err2 := ioutil.ReadAll(req.Body)
if err2 != nil {
- _ = errors.New(fmt.Sprintf("Error reading request body: %s", err2))
+ err = fmt.Errorf("Error reading request body: %s", err2)
status = http.StatusInternalServerError
return
}
fi
;;
- install-root-cert)
- set -x
- sudo cp $VAR_DATA/root-cert.pem /usr/local/share/ca-certificates/${ARVBOX_CONTAINER}-testing-cert.crt
- sudo update-ca-certificates
+ root-cert)
+ CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.pem
+ if test -n "$1" ; then
+ CERT="$1"
+ fi
+ docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT"
+ echo "Certificate copied to $CERT"
;;
devenv)
echo "host print arvbox published host"
echo "shell enter arvbox shell"
echo "open open arvbox workbench in a web browser"
+ echo "root-cert get copy of root certificate"
echo "update <config> stop, pull latest image, run"
echo "build <config> build arvbox Docker image"
echo "reboot <config> stop, build arvbox Docker image, run"
pkg-config libattr1-dev python-llfuse python-pycurl \
libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
libjson-perl nginx gitolite3 lsof libreadline-dev \
- apt-transport-https ca-certificates slurm-wlm \
+ apt-transport-https ca-certificates \
linkchecker python3-virtualenv python-virtualenv xvfb iceweasel \
libgnutls28-dev python3-dev vim cadaver cython gnupg dirmngr \
libsecret-1-dev r-base r-cran-testthat libxml2-dev pandoc \
- python3-setuptools python3-pip openjdk-8-jdk && \
+ python3-setuptools python3-pip openjdk-8-jdk bsdmainutils && \
apt-get clean
ENV RUBYVERSION_MINOR 2.3
set -u
-if ! test -s /var/lib/arvados/api_uuid_prefix ; then
- ruby -e 'puts "#{rand(2**64).to_s(36)[0,5]}"' > /var/lib/arvados/api_uuid_prefix
-fi
uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
if ! test -s /var/lib/arvados/api_secret_token ; then
+++ /dev/null
-/usr/local/lib/arvbox/logger
\ No newline at end of file
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -eux -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-
-cat > /etc/slurm-llnl/slurm.conf <<EOF
-ControlMachine=$HOSTNAME
-ControlAddr=$HOSTNAME
-AuthType=auth/munge
-DefaultStorageLoc=/var/log/slurm-llnl
-SelectType=select/cons_res
-SelectTypeParameters=CR_CPU_Memory
-SlurmUser=arvbox
-SlurmdUser=arvbox
-SlurmctldPort=7002
-SlurmctldTimeout=300
-SlurmdPort=7003
-SlurmdSpoolDir=/var/tmp/slurmd.spool
-SlurmdTimeout=300
-StateSaveLocation=/var/tmp/slurm.state
-NodeName=$HOSTNAME
-PartitionName=compute State=UP Default=YES Nodes=$HOSTNAME
-EOF
-
-mkdir -p /var/run/munge
-
-/usr/sbin/munged -f
-
-exec /usr/sbin/slurmctld -v -D
+++ /dev/null
-/usr/local/lib/arvbox/logger
\ No newline at end of file
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -eux -o pipefail
-
-exec /usr/local/lib/arvbox/runsu.sh /usr/sbin/slurmd -v -D
set -u
-if ! test -s /var/lib/arvados/sso_uuid_prefix ; then
- ruby -e 'puts "#{rand(2**64).to_s(36)[0,5]}"' > /var/lib/arvados/sso_uuid_prefix
+if ! test -s /var/lib/arvados/api_uuid_prefix ; then
+ ruby -e 'puts "x#{rand(2**64).to_s(36)[0,4]}"' > /var/lib/arvados/api_uuid_prefix
fi
-uuid_prefix=$(cat /var/lib/arvados/sso_uuid_prefix)
+uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
if ! test -s /var/lib/arvados/sso_secret_token ; then
ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/sso_secret_token
projects / arvados.git / commitdiff