From: Tom Clegg Date: Tue, 22 Dec 2020 21:48:37 +0000 (-0500) Subject: 16306: Merge branch 'master' X-Git-Tag: 2.2.0~141^2~32 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/3aaefcb3c76ff470b475d950398d01255e87712a?hp=-c 16306: Merge branch 'master' Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- 3aaefcb3c76ff470b475d950398d01255e87712a diff --combined build/run-tests.sh index 610701e6bc,595f721080..7bd4e618dd --- a/build/run-tests.sh +++ b/build/run-tests.sh @@@ -88,7 -88,7 +88,7 @@@ lib/cloud/cloudtes lib/dispatchcloud lib/dispatchcloud/container lib/dispatchcloud/scheduler - lib/dispatchcloud/ssh_executor + lib/dispatchcloud/sshexecutor lib/dispatchcloud/worker lib/mount lib/pam @@@ -162,9 -162,12 +162,12 @@@ temp_preserve clear_temp() { if [[ -z "$temp" ]]; then - # we didn't even get as far as making a temp dir + # we did not even get as far as making a temp dir : elif [[ -z "$temp_preserve" ]]; then + # Go creates readonly dirs in the module cache, which cause + # "rm -rf" to fail unless we chmod first. + chmod -R u+w "$temp" rm -rf "$temp" else echo "Leaving behind temp dirs in $temp" @@@ -200,9 -203,6 +203,6 @@@ sanity_checks() echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen" - echo -n 'virtualenv: ' - virtualenv --version \ - || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" echo -n 'ruby: ' ruby -v \ || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)" @@@ -220,9 -220,9 +220,9 @@@ echo -n 'gnutls.h: ' find /usr/include -path '*gnutls/gnutls.h' | egrep --max-count=1 . \ || fatal "No gnutls/gnutls.h. Try: apt-get install libgnutls28-dev" - echo -n 'Python2 pyconfig.h: ' - find /usr/include -path '*/python2*/pyconfig.h' | egrep --max-count=1 . \ - || fatal "No Python2 pyconfig.h. Try: apt-get install python2.7-dev" + echo -n 'virtualenv: ' + python3 -m venv -h | egrep --max-count=1 . \ + || fatal "No virtualenv. Try: apt-get install python3-venv" echo -n 'Python3 pyconfig.h: ' find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \ || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev" @@@ -389,7 -389,7 +389,7 @@@ checkpidfile() checkhealth() { svc="$1" - base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") + base=$("${VENV3DIR}/bin/python3" -c "import yaml; print(list(yaml.safe_load(open('$ARVADOS_CONFIG','r'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0])") url="$base/_health/ping" if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then echo "${url} failed" @@@ -411,7 -411,7 +411,7 @@@ start_services() if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then return 0 fi - . "$VENVDIR/bin/activate" + . "$VENV3DIR/bin/activate" echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" @@@ -424,26 -424,26 +424,26 @@@ fail=1 cd "$WORKSPACE" \ - && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ + && eval $(python3 sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ && checkpidfile api \ && checkdiscoverydoc $ARVADOS_API_HOST \ - && eval $(python sdk/python/tests/run_test_server.py start_nginx) \ + && eval $(python3 sdk/python/tests/run_test_server.py start_nginx) \ && checkpidfile nginx \ - && python sdk/python/tests/run_test_server.py start_controller \ + && python3 sdk/python/tests/run_test_server.py start_controller \ && checkpidfile controller \ && checkhealth Controller \ && checkdiscoverydoc $ARVADOS_API_HOST \ - && python sdk/python/tests/run_test_server.py start_keep_proxy \ + && python3 sdk/python/tests/run_test_server.py start_keep_proxy \ && checkpidfile keepproxy \ - && python sdk/python/tests/run_test_server.py start_keep-web \ + && python3 sdk/python/tests/run_test_server.py start_keep-web \ && checkpidfile keep-web \ && checkhealth WebDAV \ - && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py start_arv-git-httpd \ && checkpidfile arv-git-httpd \ && checkhealth GitHTTP \ - && python sdk/python/tests/run_test_server.py start_ws \ + && python3 sdk/python/tests/run_test_server.py start_ws \ && checkpidfile ws \ && export ARVADOS_TEST_PROXY_SERVICES=1 \ && (env | egrep ^ARVADOS) \ @@@ -460,15 -460,15 +460,15 @@@ stop_services() return fi unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES - . "$VENVDIR/bin/activate" || return + . "$VENV3DIR/bin/activate" || return cd "$WORKSPACE" \ - && python sdk/python/tests/run_test_server.py stop_nginx \ - && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ - && python sdk/python/tests/run_test_server.py stop_ws \ - && python sdk/python/tests/run_test_server.py stop_keep-web \ - && python sdk/python/tests/run_test_server.py stop_keep_proxy \ - && python sdk/python/tests/run_test_server.py stop_controller \ - && python sdk/python/tests/run_test_server.py stop \ + && python3 sdk/python/tests/run_test_server.py stop_nginx \ + && python3 sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py stop_ws \ + && python3 sdk/python/tests/run_test_server.py stop_keep-web \ + && python3 sdk/python/tests/run_test_server.py stop_keep_proxy \ + && python3 sdk/python/tests/run_test_server.py stop_controller \ + && python3 sdk/python/tests/run_test_server.py stop \ && all_services_stopped=1 deactivate unset ARVADOS_CONFIG @@@ -541,16 -541,16 +541,16 @@@ setup_ruby_environment() tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)" PATH="$tmpdir_gem_home/bin:$PATH" - export GEM_PATH="$tmpdir_gem_home" + export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)" echo "Will install dependencies to $(gem env gemdir)" - echo "Will install arvados gems to $tmpdir_gem_home" + echo "Will install bundler and arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" - bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" + bundle="$tmpdir_gem_home/bin/bundle" ( export HOME=$GEMHOME bundlers="$(gem list --details bundler)" - versions=(1.11.0 1.17.3 2.0.2) + versions=(1.16.6 1.17.3 2.0.2) for v in ${versions[@]}; do if ! echo "$bundlers" | fgrep -q "($v)"; then gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) @@@ -578,17 -578,12 +578,12 @@@ gem_uninstall_if_exists() setup_virtualenv() { local venvdest="$1"; shift - if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then - virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed" + if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip3" ]]; then + python3 -m venv "$@" "$venvdest" || fatal "virtualenv $venvdest failed" elif [[ -n "$short" ]]; then return fi - if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then - # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7,<8' - else - "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' - fi + "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' } initialize() { @@@ -610,7 -605,7 +605,7 @@@ fi # Set up temporary install dirs (unless existing dirs were supplied) - for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS + for tmpdir in VENV3DIR GOPATH GEMHOME PERLINSTALLBASE R_LIBS do if [[ -z "${!tmpdir}" ]]; then eval "$tmpdir"="$temp/$tmpdir" @@@ -651,34 -646,25 +646,25 @@@ install_env() go mod download || fatal "Go deps failed" which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" - setup_virtualenv "$VENVDIR" --python python2.7 - . "$VENVDIR/bin/activate" + setup_virtualenv "$VENV3DIR" + . "$VENV3DIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. + # pdoc3 needed to generate the Python SDK documentation. ( set -e - "${VENVDIR}/bin/pip" install PyYAML - "${VENV3DIR}/bin/pip" install PyYAML + "${VENV3DIR}/bin/pip3" install wheel + "${VENV3DIR}/bin/pip3" install PyYAML + "${VENV3DIR}/bin/pip3" install httplib2 + "${VENV3DIR}/bin/pip3" install future + "${VENV3DIR}/bin/pip3" install google-api-python-client + "${VENV3DIR}/bin/pip3" install ciso8601 + "${VENV3DIR}/bin/pip3" install pycurl + "${VENV3DIR}/bin/pip3" install ws4py + "${VENV3DIR}/bin/pip3" install pdoc3 cd "$WORKSPACE/sdk/python" - python setup.py install + python3 setup.py install ) || fatal "installing PyYAML and sdk/python failed" - - # Deactivate Python 2 virtualenv - deactivate - - # If Python 3 is available, set up its virtualenv in $VENV3DIR. - # Otherwise, skip dependent tests. - PYTHON3=$(which python3) - if [[ ${?} = 0 ]]; then - setup_virtualenv "$VENV3DIR" --python python3 - else - PYTHON3= - cat >&2 </dev/null; then deactivate; fi - if ! . "$VENVDIR/bin/activate" + if ! . "$VENV3DIR/bin/activate" then result=1 elif [[ "$2" == "go" ]] @@@ -823,12 -809,12 +809,12 @@@ check_arvados_config() # Create config file. The run_test_server script requires PyYAML, # so virtualenv needs to be active. Downstream steps like # workbench install which require a valid config.yml. - if [[ ! -s "$VENVDIR/bin/activate" ]] ; then + if [[ ! -s "$VENV3DIR/bin/activate" ]] ; then install_env fi - . "$VENVDIR/bin/activate" + . "$VENV3DIR/bin/activate" cd "$WORKSPACE" - eval $(python sdk/python/tests/run_test_server.py setup_config) + eval $(python3 sdk/python/tests/run_test_server.py setup_config) deactivate fi } @@@ -847,7 -833,7 +833,7 @@@ do_install_once() result= if which deactivate >/dev/null; then deactivate; fi - if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then + if [[ "$1" != "env" ]] && ! . "$VENV3DIR/bin/activate"; then result=1 elif [[ "$2" == "go" ]] then @@@ -867,10 -853,10 +853,10 @@@ # install" ensures that we've actually installed the local package # we just built. cd "$WORKSPACE/$1" \ - && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ + && "${3}python3" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip3" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip3" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" @@@ -950,7 -936,7 +936,7 @@@ install_services/api() # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ + && test_database=$("${VENV3DIR}/bin/python3" -c "import yaml; print(yaml.safe_load(open('$ARVADOS_CONFIG','r'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname'])") \ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@@ -959,7 -945,7 +945,7 @@@ if [[ ! -e "$cert.pem" || "$(date -r "$cert.pem" +%s)" -lt 1512659226 ]]; then ( dir="$WORKSPACE/services/api/tmp" - set -ex + set -e openssl req -newkey rsa:2048 -nodes -subj '/C=US/ST=State/L=City/CN=localhost' -out "$cert.csr" -keyout "$cert.key" table(table table-bordered table-condensed). |||\5=. Appropriate for| - ||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation| + ||_. Setup difficulty|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation| |"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes| + |"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes| + |"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes| |"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes| +|"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes| - |"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no| + |"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no| |"Cluster Operation Subscription supported by Curii":mailto:info@curii.com|N/A ^3^|yes|yes|yes|yes|yes| diff --combined lib/boot/postgresql.go index fc23eb9132,7661c6b587..e45c4e1686 --- a/lib/boot/postgresql.go +++ b/lib/boot/postgresql.go @@@ -36,13 -36,9 +36,13 @@@ func (runPostgreSQL) Run(ctx context.Co return err } + if super.ClusterType == "production" { + return nil + } + iamroot := false if u, err := user.Current(); err != nil { - return fmt.Errorf("user.Current(): %s", err) + return fmt.Errorf("user.Current(): %w", err) } else if u.Uid == "0" { iamroot = true } @@@ -65,7 -61,7 +65,7 @@@ if err != nil { return fmt.Errorf("user.Lookup(\"postgres\"): %s", err) } - postgresUid, err := strconv.Atoi(postgresUser.Uid) + postgresUID, err := strconv.Atoi(postgresUser.Uid) if err != nil { return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid) } @@@ -81,7 -77,7 +81,7 @@@ if err != nil { return err } - err = os.Chown(datadir, postgresUid, 0) + err = os.Chown(datadir, postgresUID, 0) if err != nil { return err } diff --combined lib/boot/seed.go index 1f07601a09,2afccc45b6..1f6cb764e0 --- a/lib/boot/seed.go +++ b/lib/boot/seed.go @@@ -20,12 -20,13 +20,16 @@@ func (seedDatabase) Run(ctx context.Con if err != nil { return err } + if super.ClusterType == "production" { + return nil + } err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "rake", "db:setup") if err != nil { return err } + err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb") + if err != nil { + return err + } return nil } diff --combined lib/boot/supervisor.go index 138c802e18,1e8e83ff3b..20576b6b97 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@@ -14,7 -14,6 +14,7 @@@ import "io" "io/ioutil" "net" + "net/url" "os" "os/exec" "os/signal" @@@ -55,9 -54,7 +55,9 @@@ type Supervisor struct tasksReady map[string]chan bool waitShutdown sync.WaitGroup + bindir string tempdir string + wwwtempdir string configfile string environ []string // for child processes } @@@ -134,26 -131,13 +134,26 @@@ func (super *Supervisor) run(cfg *arvad return err } - super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") - if err != nil { - return err - } - defer os.RemoveAll(super.tempdir) - if err := os.Mkdir(filepath.Join(super.tempdir, "bin"), 0755); err != nil { - return err + // Choose bin and temp dirs: /var/lib/arvados/... in + // production, transient tempdir otherwise. + if super.ClusterType == "production" { + // These dirs have already been created by + // "arvados-server install" (or by extracting a + // package). + super.tempdir = "/var/lib/arvados/tmp" + super.wwwtempdir = "/var/lib/arvados/wwwtmp" + super.bindir = "/var/lib/arvados/bin" + } else { + super.tempdir, err = ioutil.TempDir("", "arvados-server-boot-") + if err != nil { + return err + } + defer os.RemoveAll(super.tempdir) + super.wwwtempdir = super.tempdir + super.bindir = filepath.Join(super.tempdir, "bin") + if err := os.Mkdir(super.bindir, 0755); err != nil { + return err + } } // Fill in any missing config keys, and write the resulting @@@ -182,10 -166,7 +182,10 @@@ super.setEnv("ARVADOS_CONFIG", super.configfile) super.setEnv("RAILS_ENV", super.ClusterType) super.setEnv("TMPDIR", super.tempdir) - super.prependEnv("PATH", super.tempdir+"/bin:/var/lib/arvados/bin:") + super.prependEnv("PATH", "/var/lib/arvados/bin:") + if super.ClusterType != "production" { + super.prependEnv("PATH", super.tempdir+"/bin:") + } super.cluster, err = cfg.GetCluster("") if err != nil { @@@ -201,9 -182,7 +201,9 @@@ "PID": os.Getpid(), }) - if super.SourceVersion == "" { + if super.SourceVersion == "" && super.ClusterType == "production" { + // don't need SourceVersion + } else if super.SourceVersion == "" { // Find current source tree version. var buf bytes.Buffer err = super.RunProgram(super.ctx, ".", &buf, nil, "git", "diff", "--shortstat") @@@ -245,15 -224,15 +245,15 @@@ runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV}, runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{runPostgreSQL{}}}, installPassenger{src: "services/api"}, - runPassenger{src: "services/api", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}}, + runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, runPostgreSQL{}, installPassenger{src: "services/api"}}}, installPassenger{src: "apps/workbench", depends: []supervisedTask{installPassenger{src: "services/api"}}}, // dependency ensures workbench doesn't delay api startup - runPassenger{src: "apps/workbench", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, + runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench"}}}, seedDatabase{}, } if super.ClusterType != "test" { tasks = append(tasks, - runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.Controller}, - runGoProgram{src: "services/keep-balance"}, + runServiceCommand{name: "dispatch-cloud", svc: super.cluster.Services.DispatchCloud}, + runGoProgram{src: "services/keep-balance", svc: super.cluster.Services.Keepbalance}, ) } super.tasksReady = map[string]chan bool{} @@@ -403,11 -382,9 +403,11 @@@ func dedupEnv(in []string) []string func (super *Supervisor) installGoProgram(ctx context.Context, srcpath string) (string, error) { _, basename := filepath.Split(srcpath) - bindir := filepath.Join(super.tempdir, "bin") - binfile := filepath.Join(bindir, basename) - err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) + binfile := filepath.Join(super.bindir, basename) + if super.ClusterType == "production" { + return binfile, nil + } + err := super.RunProgram(ctx, filepath.Join(super.SourcePath, srcpath), nil, []string{"GOBIN=" + super.bindir}, "go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+super.SourceVersion+" -X main.version="+super.SourceVersion) return binfile, err } @@@ -424,19 -401,10 +424,19 @@@ func (super *Supervisor) setupRubyEnv( "GEM_PATH=", }) gem := "gem" - if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil { + if _, err := os.Stat("/var/lib/arvados/bin/gem"); err == nil || super.ClusterType == "production" { gem = "/var/lib/arvados/bin/gem" } cmd := exec.Command(gem, "env", "gempath") + if super.ClusterType == "production" { + cmd.Args = append([]string{"sudo", "-u", "www-data", "-E", "HOME=/var/www"}, cmd.Args...) + path, err := exec.LookPath("sudo") + if err != nil { + return fmt.Errorf("LookPath(\"sudo\"): %w", err) + } + cmd.Path = path + } + cmd.Stderr = super.Stderr cmd.Env = super.environ buf, err := cmd.Output() // /var/lib/arvados/.gem/ruby/2.5.0/bin:... if err != nil || len(buf) == 0 { @@@ -470,9 -438,9 +470,9 @@@ func (super *Supervisor) lookPath(prog return prog } - // Run prog with args, using dir as working directory. If ctx is - // cancelled while the child is running, RunProgram terminates the - // child, waits for it to exit, then returns. + // RunProgram runs prog with args, using dir as working directory. If ctx is + // cancelled while the child is running, RunProgram terminates the child, waits + // for it to exit, then returns. // // Child's environment will have our env vars, plus any given in env. // @@@ -483,34 -451,17 +483,34 @@@ func (super *Supervisor) RunProgram(ct super.logger.WithField("command", cmdline).WithField("dir", dir).Info("executing") logprefix := prog - if logprefix == "setuidgid" && len(args) >= 3 { - logprefix = args[2] - } - logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/") - if logprefix == "bundle" && len(args) > 2 && args[0] == "exec" { - logprefix = args[1] - } else if logprefix == "arvados-server" && len(args) > 1 { - logprefix = args[0] - } - if !strings.HasPrefix(dir, "/") { - logprefix = dir + ": " + logprefix + { + if logprefix == "setuidgid" && len(args) >= 3 { + logprefix = args[2] + } + innerargs := args + if logprefix == "sudo" { + for i := 0; i < len(args); i++ { + if args[i] == "-u" { + i++ + } else if args[i] == "-E" || strings.Contains(args[i], "=") { + } else { + logprefix = args[i] + innerargs = args[i+1:] + break + } + } + } + logprefix = strings.TrimPrefix(logprefix, "/var/lib/arvados/bin/") + logprefix = strings.TrimPrefix(logprefix, super.tempdir+"/bin/") + if logprefix == "bundle" && len(innerargs) > 2 && innerargs[0] == "exec" { + _, dirbase := filepath.Split(dir) + logprefix = innerargs[1] + "@" + dirbase + } else if logprefix == "arvados-server" && len(args) > 1 { + logprefix = args[0] + } + if !strings.HasPrefix(dir, "/") { + logprefix = dir + ": " + logprefix + } } cmd := exec.Command(super.lookPath(prog), args...) @@@ -650,30 -601,34 +650,30 @@@ func (super *Supervisor) autofillConfig } if len(svc.InternalURLs) == 0 { svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{ - arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{}, + {Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: {}, } } } - if cluster.SystemRootToken == "" { - cluster.SystemRootToken = randomHexString(64) - } - if cluster.ManagementToken == "" { - cluster.ManagementToken = randomHexString(64) - } - if cluster.API.RailsSessionSecretToken == "" { - cluster.API.RailsSessionSecretToken = randomHexString(64) - } - if cluster.Collections.BlobSigningKey == "" { - cluster.Collections.BlobSigningKey = randomHexString(64) - } - if cluster.Users.AnonymousUserToken == "" { - cluster.Users.AnonymousUserToken = randomHexString(64) - } - - if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" { - buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) - if err != nil { - return err - } - cluster.Containers.DispatchPrivateKey = string(buf) - } if super.ClusterType != "production" { + if cluster.SystemRootToken == "" { + cluster.SystemRootToken = randomHexString(64) + } + if cluster.ManagementToken == "" { + cluster.ManagementToken = randomHexString(64) + } + if cluster.API.RailsSessionSecretToken == "" { + cluster.API.RailsSessionSecretToken = randomHexString(64) + } + if cluster.Collections.BlobSigningKey == "" { + cluster.Collections.BlobSigningKey = randomHexString(64) + } + if cluster.Containers.DispatchPrivateKey == "" { + buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch")) + if err != nil { + return err + } + cluster.Containers.DispatchPrivateKey = string(buf) + } cluster.TLS.Insecure = true } if super.ClusterType == "test" { @@@ -743,10 -698,11 +743,10 @@@ func internalPort(svc arvados.Service) return "", errors.New("internalPort() doesn't work with multiple InternalURLs") } for u := range svc.InternalURLs { - if _, p, err := net.SplitHostPort(u.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(u) + if p := u.Port(); p != "" { return p, nil - } else if u.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "ws" { return "443", nil } else { return "80", nil @@@ -756,10 -712,11 +756,10 @@@ } func externalPort(svc arvados.Service) (string, error) { - if _, p, err := net.SplitHostPort(svc.ExternalURL.Host); err != nil { - return "", err - } else if p != "" { + u := url.URL(svc.ExternalURL) + if p := u.Port(); p != "" { return p, nil - } else if svc.ExternalURL.Scheme == "https" { + } else if u.Scheme == "https" || u.Scheme == "wss" { return "443", nil } else { return "80", nil diff --combined lib/install/deps.go index da45b393bf,342ef03a7f..cc9595db64 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@@ -14,8 -14,6 +14,8 @@@ import "io" "os" "os/exec" + "os/user" + "path/filepath" "strconv" "strings" "syscall" @@@ -26,17 -24,13 +26,17 @@@ "github.com/lib/pq" ) -var Command cmd.Handler = installCommand{} +var Command cmd.Handler = &installCommand{} const devtestDatabasePassword = "insecure_arvados_test" -type installCommand struct{} +type installCommand struct { + ClusterType string + SourcePath string + PackageVersion string +} -func (installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { +func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { logger := ctxlog.New(stderr, "text", "info") ctx := ctxlog.Context(context.Background(), logger) ctx, cancel := context.WithCancel(ctx) @@@ -52,9 -46,7 +52,9 @@@ flags := flag.NewFlagSet(prog, flag.ContinueOnError) flags.SetOutput(stderr) versionFlag := flags.Bool("version", false, "Write version information to stdout and exit 0") - clusterType := flags.String("type", "production", "cluster `type`: development, test, or production") + flags.StringVar(&inst.ClusterType, "type", "production", "cluster `type`: development, test, production, or package") + flags.StringVar(&inst.SourcePath, "source", "/arvados", "source tree location (required for -type=package)") + flags.StringVar(&inst.PackageVersion, "package-version", "0.0.0", "version string to embed in executable files") err = flags.Parse(args) if err == flag.ErrHelp { err = nil @@@ -63,23 -55,18 +63,23 @@@ return 2 } else if *versionFlag { return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr) + } else if len(flags.Args()) > 0 { + err = fmt.Errorf("unrecognized command line arguments: %v", flags.Args()) + return 2 } - var dev, test, prod bool - switch *clusterType { + var dev, test, prod, pkg bool + switch inst.ClusterType { case "development": dev = true case "test": test = true case "production": prod = true + case "package": + pkg = true default: - err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', or 'production')", *clusterType) + err = fmt.Errorf("invalid cluster type %q (must be 'development', 'test', 'production', or 'package')", inst.ClusterType) return 2 } @@@ -109,67 -96,74 +109,73 @@@ } } - if dev || test { - debs := []string{ + pkgs := prodpkgs(osv) + + if pkg { + pkgs = append(pkgs, + "dpkg-dev", + "rsync", + ) + } + + if dev || test || pkg { + pkgs = append(pkgs, + "automake", "bison", "bsdmainutils", "build-essential", - "ca-certificates", "cadaver", - "cython", + "curl", + "cython3", "daemontools", // lib/boot uses setuidgid to drop privileges when running as root "default-jdk-headless", "default-jre-headless", - "fuse", "gettext", - "git", - "gitolite3", - "graphviz", - "haveged", "iceweasel", "libattr1-dev", "libcrypt-ssleay-perl", - "libcrypt-ssleay-perl", - "libcurl3-gnutls", - "libcurl4-openssl-dev", "libfuse-dev", "libgnutls28-dev", "libjson-perl", - "libjson-perl", "libpam-dev", "libpcre3-dev", - "libpython2.7-dev", + "libpq-dev", "libreadline-dev", "libssl-dev", "libwww-perl", "libxml2-dev", - "libxslt1.1", + "libxslt1-dev", "linkchecker", "lsof", + "make", "net-tools", - "nginx", "pandoc", "perl-modules", "pkg-config", "postgresql", "postgresql-contrib", "python3-dev", - "python-epydoc", + "python3-venv", + "python3-virtualenv", "r-base", "r-cran-testthat", + "r-cran-devtools", + "r-cran-knitr", + "r-cran-markdown", + "r-cran-roxygen2", + "r-cran-xml", "sudo", - "virtualenv", "wget", "xvfb", - "zlib1g-dev", - } + ) switch { case osv.Debian && osv.Major >= 10: - debs = append(debs, "libcurl4") + pkgs = append(pkgs, "libcurl4") default: - debs = append(debs, "libcurl3") + pkgs = append(pkgs, "libcurl3") } cmd := exec.CommandContext(ctx, "apt-get", "install", "--yes", "--no-install-recommends") - cmd.Args = append(cmd.Args, debs...) + cmd.Args = append(cmd.Args, pkgs...) cmd.Env = append(os.Environ(), "DEBIAN_FRONTEND=noninteractive") cmd.Stdout = stdout cmd.Stderr = stderr @@@ -180,36 -174,20 +186,36 @@@ } os.Mkdir("/var/lib/arvados", 0755) + os.Mkdir("/var/lib/arvados/tmp", 0700) + if prod || pkg { + os.Mkdir("/var/lib/arvados/wwwtmp", 0700) + u, er := user.Lookup("www-data") + if er != nil { + err = fmt.Errorf("user.Lookup(%q): %w", "www-data", er) + return 1 + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(u.Gid) + err = os.Chown("/var/lib/arvados/wwwtmp", uid, gid) + if err != nil { + return 1 + } + } rubyversion := "2.5.7" if haverubyversion, err := exec.Command("/var/lib/arvados/bin/ruby", "-v").CombinedOutput(); err == nil && bytes.HasPrefix(haverubyversion, []byte("ruby "+rubyversion)) { logger.Print("ruby " + rubyversion + " already installed") } else { err = runBash(` -mkdir -p /var/lib/arvados/tmp tmp=/var/lib/arvados/tmp/ruby-`+rubyversion+` trap "rm -r ${tmp}" ERR wget --progress=dot:giga -O- https://cache.ruby-lang.org/pub/ruby/2.5/ruby-`+rubyversion+`.tar.gz | tar -C /var/lib/arvados/tmp -xzf - cd ${tmp} -./configure --disable-install-doc --prefix /var/lib/arvados -make -j4 +./configure --disable-install-static-library --enable-shared --disable-install-doc --prefix /var/lib/arvados +make -j8 make install -/var/lib/arvados/bin/gem install bundler +/var/lib/arvados/bin/gem install bundler --no-ri --no-rdoc +# "gem update --system" can be removed when we use ruby ≥2.6.3: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html +/var/lib/arvados/bin/gem update --system --no-ri --no-rdoc rm -r ${tmp} `, stdout, stderr) if err != nil { @@@ -231,9 -209,7 +237,9 @@@ ln -sf /var/lib/arvados/go/bin/* /usr/l return 1 } } + } + if !prod && !pkg { pjsversion := "1.9.8" if havepjsversion, err := exec.Command("/usr/local/bin/phantomjs", "--version").CombinedOutput(); err == nil && string(havepjsversion) == "1.9.8\n" { logger.Print("phantomjs " + pjsversion + " already installed") @@@ -282,6 -258,7 +288,6 @@@ ln -sf /var/lib/arvados/node-${NJS}-lin } else { err = runBash(` G=`+gradleversion+` -mkdir -p /var/lib/arvados/tmp zip=/var/lib/arvados/tmp/gradle-${G}-bin.zip trap "rm ${zip}" ERR wget --progress=dot:giga -O${zip} https://services.gradle.org/distributions/gradle-${G}-bin.zip @@@ -316,10 -293,10 +322,10 @@@ rm ${zip DataDirectory string LogFile string } - if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil { + if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil { err = fmt.Errorf("pg_lsclusters: %s", err2) return 1 - } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 { + } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 { logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters)) } else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil { err = fmt.Errorf("error parsing pg_lsclusters output: %s", err) @@@ -415,104 -392,12 +421,104 @@@ } } + if prod || pkg { + // Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/ + for dstdir, srcdir := range map[string]string{ + "railsapi": "services/api", + "workbench1": "apps/workbench", + } { + fmt.Fprintf(stderr, "building %s...\n", srcdir) + cmd := exec.Command("rsync", + "-a", "--no-owner", "--delete-after", "--delete-excluded", + "--exclude", "/coverage", + "--exclude", "/log", + "--exclude", "/tmp", + "--exclude", "/vendor", + "./", "/var/lib/arvados/"+dstdir+"/") + cmd.Dir = filepath.Join(inst.SourcePath, srcdir) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + for _, cmdline := range [][]string{ + {"mkdir", "-p", "log", "tmp", ".bundle", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger"}, + {"touch", "log/production.log"}, + {"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.gem", "/var/www/.bundle", "/var/www/.passenger", "log", "tmp", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:1.16.6", "bundler:1.17.3", "bundler:2.0.2"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "build-native-support"}, + {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "install-standalone-runtime"}, + } { + cmd = exec.Command(cmdline[0], cmdline[1:]...) + cmd.Env = append([]string{}, os.Environ()...) + cmd.Dir = "/var/lib/arvados/" + dstdir + cmd.Stdout = stdout + cmd.Stderr = stderr + fmt.Fprintf(stderr, "... %s\n", cmd.Args) + err = cmd.Run() + if err != nil { + return 1 + } + } + cmd = exec.Command("sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "exec", "passenger-config", "validate-install") + cmd.Dir = "/var/lib/arvados/" + dstdir + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil && !strings.Contains(err.Error(), "exit status 2") { + // Exit code 2 indicates there were warnings (like + // "other passenger installations have been detected", + // which we can't expect to avoid) but no errors. + // Other non-zero exit codes (1, 9) indicate errors. + return 1 + } + } + + // Install Go programs to /var/lib/arvados/bin/ + for _, srcdir := range []string{ + "cmd/arvados-client", + "cmd/arvados-server", + "services/arv-git-httpd", + "services/crunch-dispatch-local", + "services/crunch-dispatch-slurm", + "services/health", + "services/keep-balance", + "services/keep-web", + "services/keepproxy", + "services/keepstore", + "services/ws", + } { + fmt.Fprintf(stderr, "building %s...\n", srcdir) + cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion) + cmd.Env = append([]string{"GOBIN=/var/lib/arvados/bin"}, os.Environ()...) + cmd.Dir = filepath.Join(inst.SourcePath, srcdir) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + } + + // Copy assets from source tree to /var/lib/arvados/share + cmd := exec.Command("install", "-v", "-t", "/var/lib/arvados/share", filepath.Join(inst.SourcePath, "sdk/python/tests/nginx.conf")) + cmd.Stdout = stdout + cmd.Stderr = stderr + err = cmd.Run() + if err != nil { + return 1 + } + } + return 0 } type osversion struct { Debian bool Ubuntu bool + Centos bool Major int } @@@ -550,8 -435,6 +556,8 @@@ func identifyOS() (osversion, error) osv.Ubuntu = true case "debian": osv.Debian = true + case "centos": + osv.Centos = true default: return osv, fmt.Errorf("unsupported ID in /etc/os-release: %q", kv["ID"]) } @@@ -586,54 -469,3 +592,54 @@@ func runBash(script string, stdout, std cmd.Stderr = stderr return cmd.Run() } + +func prodpkgs(osv osversion) []string { + pkgs := []string{ + "ca-certificates", + "curl", + "fuse", + "git", + "gitolite3", + "graphviz", + "haveged", + "libcurl3-gnutls", + "libxslt1.1", + "nginx", + "python", + "sudo", + } + if osv.Debian || osv.Ubuntu { + if osv.Debian && osv.Major == 8 { + pkgs = append(pkgs, "libgnutls-deb0-28") // sdk/cwl + } else if osv.Debian && osv.Major >= 10 || osv.Ubuntu && osv.Major >= 16 { + pkgs = append(pkgs, "python3-distutils") // sdk/cwl + } + return append(pkgs, + "g++", + "libcurl4-openssl-dev", // services/api + "libpq-dev", + "libpython2.7", // services/fuse + "mime-support", // keep-web + "zlib1g-dev", // services/api + ) + } else if osv.Centos { + return append(pkgs, + "fuse-libs", // services/fuse + "gcc", + "gcc-c++", + "libcurl-devel", // services/api + "mailcap", // keep-web + "postgresql-devel", // services/api + ) + } else { + panic("os version not supported") + } +} + +func ProductionDependencies() ([]string, error) { + osv, err := identifyOS() + if err != nil { + return nil, err + } + return prodpkgs(osv), nil +}