From: Tom Clegg Date: Thu, 5 May 2022 15:25:13 +0000 (-0400) Subject: 18794: Merge branch 'main' X-Git-Tag: 2.5.0~184^2~6 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/ac2ea4ef187f7df369551a8cfa047fed8a1a2ba9?hp=3b2568c566f7db8f56ba3425b21a4f7fcb46a083 18794: Merge branch 'main' Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb index 237cf27555..86e3cdd91d 100644 --- a/apps/workbench/app/controllers/work_units_controller.rb +++ b/apps/workbench/app/controllers/work_units_controller.rb @@ -83,6 +83,7 @@ class WorkUnitsController < ApplicationController attrs['properties'] = {'template_uuid' => template_uuid} attrs['priority'] = 1 attrs['state'] = "Uncommitted" + attrs['use_existing'] = false # required attrs['container_image'] = "arvados/jobs" diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb index 8b89d57ee9..fd30be9462 100644 --- a/apps/workbench/app/models/repository.rb +++ b/apps/workbench/app/models/repository.rb @@ -81,7 +81,7 @@ class Repository < ArvadosBase FileUtils.mkdir_p Rails.configuration.Workbench.RepositoryCache [['git', 'init', '--bare', workdir], ].each do |cmd| - system *cmd + system(*cmd, in: "/dev/null") raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0 end end @@ -96,14 +96,15 @@ class Repository < ArvadosBase 'http.sslVerify', Rails.configuration.TLS.Insecure ? 'false' : 'true'], ].each do |cmd| - system *cmd + system(*cmd, in: "/dev/null") raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0 end env = {}. merge(ENV). - merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token]) + merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token], + 'GIT_TERMINAL_PROMPT' => '0') cmd = ['git', '--git-dir', @workdir] + gitcmd - io = IO.popen(env, cmd, err: [:child, :out]) + io = IO.popen(env, cmd, err: [:child, :out], in: "/dev/null") output = io.read io.close # "If [io] is opened by IO.popen, close sets $?." --ruby 2.2.1 docs diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb index 1fd9d90084..4795486c55 100644 --- a/apps/workbench/test/integration/repositories_browse_test.rb +++ b/apps/workbench/test/integration/repositories_browse_test.rb @@ -18,9 +18,6 @@ class RepositoriesTest < ActionDispatch::IntegrationTest test "browse using arv-git-http" do repo = api_fixture('repositories')['foo'] - Repository.any_instance. - stubs(:http_fetch_url). - returns "#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}/#{repo['name']}.git" commit_sha1 = '1de84a854e2b440dc53bf42f8548afa4c17da332' visit page_with_token('active', "/repositories/#{repo['uuid']}/commit/#{commit_sha1}") assert_text "Date: Tue Mar 18 15:55:28 2014 -0400" diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 9b09b7fae0..adcbab8a09 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -244,7 +244,7 @@ package_go_binary cmd/arvados-server arvados-dispatch-cloud "$FORMAT" "$ARCH" \ "Arvados cluster cloud dispatch" package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \ "Dispatch Arvados containers to an LSF cluster" -package_go_binary services/arv-git-httpd arvados-git-httpd "$FORMAT" "$ARCH" \ +package_go_binary cmd/arvados-server arvados-git-httpd "$FORMAT" "$ARCH" \ "Provide authenticated http access to Arvados-hosted git repositories" package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \ "Dispatch Crunch containers on the local system" @@ -262,7 +262,7 @@ package_go_binary cmd/arvados-server keepproxy "$FORMAT" "$ARCH" \ "Make a Keep cluster accessible to clients that are not on the LAN" package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \ "Keep storage daemon, accessible to clients on the LAN" -package_go_binary services/keep-web keep-web "$FORMAT" "$ARCH" \ +package_go_binary cmd/arvados-server keep-web "$FORMAT" "$ARCH" \ "Static web hosting service for user data stored in Arvados Keep" package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \ "Arvados Websocket server" diff --git a/build/run-tests.sh b/build/run-tests.sh index 67c54c98b0..d517e7e2c4 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -40,7 +40,7 @@ sdk/python_test="--test-suite tests.test_keep_locator" Restrict Python SDK tests to the given class apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb" Restrict Workbench tests to the given file -services/arv-git-httpd_test="-check.vv" +services/githttpd_test="-check.vv" Show all log messages, even when tests pass (also works with services/keepstore_test etc.) ARVADOS_DEBUG=1 @@ -92,7 +92,7 @@ lib/mount lib/pam lib/service services/api -services/arv-git-httpd +services/githttpd services/crunchstat services/dockercleaner services/fuse @@ -410,7 +410,7 @@ start_services() { return 0 fi . "$VENV3DIR/bin/activate" - echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' + echo 'Starting API, controller, keepproxy, keep-web, githttpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" fi @@ -438,8 +438,8 @@ start_services() { && python3 sdk/python/tests/run_test_server.py start_keep-web \ && checkpidfile keep-web \ && checkhealth WebDAV \ - && python3 sdk/python/tests/run_test_server.py start_arv-git-httpd \ - && checkpidfile arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py start_githttpd \ + && checkpidfile githttpd \ && checkhealth GitHTTP \ && python3 sdk/python/tests/run_test_server.py start_ws \ && checkpidfile ws \ @@ -461,7 +461,7 @@ stop_services() { . "$VENV3DIR/bin/activate" || return cd "$WORKSPACE" \ && python3 sdk/python/tests/run_test_server.py stop_nginx \ - && python3 sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python3 sdk/python/tests/run_test_server.py stop_githttpd \ && python3 sdk/python/tests/run_test_server.py stop_ws \ && python3 sdk/python/tests/run_test_server.py stop_keep-web \ && python3 sdk/python/tests/run_test_server.py stop_keep_proxy \ @@ -985,7 +985,7 @@ pythonstuff=( ) declare -a gostuff -gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u)) +gostuff=($(cd "$WORKSPACE" && git ls-files | grep '\.go$' | sed -e 's/\/[^\/]*$//' | sort -u)) install_apps/workbench() { cd "$WORKSPACE/apps/workbench" \ @@ -1083,7 +1083,6 @@ install_deps() { do_install sdk/python pip "${VENV3DIR}/bin/" do_install sdk/ruby do_install services/api - do_install services/arv-git-httpd go do_install services/keepproxy go do_install services/keep-web go } diff --git a/services/arv-git-httpd/arvados-git-httpd.service b/cmd/arvados-server/arvados-git-httpd.service similarity index 78% rename from services/arv-git-httpd/arvados-git-httpd.service rename to cmd/arvados-server/arvados-git-httpd.service index 6c2b3bc58d..b45587ffc0 100644 --- a/services/arv-git-httpd/arvados-git-httpd.service +++ b/cmd/arvados-server/arvados-git-httpd.service @@ -6,13 +6,17 @@ Description=Arvados git server Documentation=https://doc.arvados.org/ After=network.target +AssertPathExists=/etc/arvados/config.yml # systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section StartLimitIntervalSec=0 [Service] Type=notify +EnvironmentFile=-/etc/arvados/environment ExecStart=/usr/bin/arvados-git-httpd +# Set a reasonable default for the open file limit +LimitNOFILE=65536 Restart=always RestartSec=1 diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go index 9e02d45b67..342446a811 100644 --- a/cmd/arvados-server/cmd.go +++ b/cmd/arvados-server/cmd.go @@ -22,6 +22,8 @@ import ( "git.arvados.org/arvados.git/lib/lsf" "git.arvados.org/arvados.git/lib/recovercollection" "git.arvados.org/arvados.git/sdk/go/health" + "git.arvados.org/arvados.git/services/githttpd" + keepweb "git.arvados.org/arvados.git/services/keep-web" "git.arvados.org/arvados.git/services/keepproxy" "git.arvados.org/arvados.git/services/keepstore" "git.arvados.org/arvados.git/services/ws" @@ -43,8 +45,10 @@ var ( "crunch-run": crunchrun.Command, "dispatch-cloud": dispatchcloud.Command, "dispatch-lsf": lsf.DispatchCommand, + "git-httpd": githttpd.Command, "install": install.Command, "init": install.InitCommand, + "keep-web": keepweb.Command, "keepproxy": keepproxy.Command, "keepstore": keepstore.Command, "recover-collection": recovercollection.Command, diff --git a/services/keep-web/keep-web.service b/cmd/arvados-server/keep-web.service similarity index 81% rename from services/keep-web/keep-web.service rename to cmd/arvados-server/keep-web.service index cb5fdf84fc..c0e193d6d8 100644 --- a/services/keep-web/keep-web.service +++ b/cmd/arvados-server/keep-web.service @@ -3,15 +3,17 @@ # SPDX-License-Identifier: AGPL-3.0 [Unit] -Description=Arvados Keep web gateway +Description=Arvados Keep WebDAV and S3 gateway Documentation=https://doc.arvados.org/ After=network.target +AssertPathExists=/etc/arvados/config.yml # systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section StartLimitIntervalSec=0 [Service] Type=notify +EnvironmentFile=-/etc/arvados/environment ExecStart=/usr/bin/keep-web # Set a reasonable default for the open file limit LimitNOFILE=65536 diff --git a/doc/_config.yml b/doc/_config.yml index 9dd7f40529..f3ab1a5af0 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -50,6 +50,7 @@ navbar: - user/topics/storage-classes.html.textile.liquid - Data Analysis with Workflows: - user/cwl/arvados-vscode-training.html.md.liquid + - user/cwl/rnaseq-cwl-training.html.textile.liquid - user/cwl/cwl-runner.html.textile.liquid - user/cwl/cwl-run-options.html.textile.liquid - user/tutorials/writing-cwl-workflow.html.textile.liquid diff --git a/doc/admin/management-token.html.textile.liquid b/doc/admin/management-token.html.textile.liquid index abdd8db734..a4939b740c 100644 --- a/doc/admin/management-token.html.textile.liquid +++ b/doc/admin/management-token.html.textile.liquid @@ -21,7 +21,7 @@ h2. API server and other services The following services also support monitoring. * API server -* arv-git-httpd +* arvados-git-httpd * controller * keep-balance * keepproxy diff --git a/doc/architecture/index.html.textile.liquid b/doc/architecture/index.html.textile.liquid index dddcd05073..09d593db2b 100644 --- a/doc/architecture/index.html.textile.liquid +++ b/doc/architecture/index.html.textile.liquid @@ -19,7 +19,7 @@ Located in @arvados/services@ except for Workbench which is located in @arvados/ table(table table-bordered table-condensed). |_. Component|_. Description| |api|The API server is the core of Arvados. It is backed by a Postgres database and manages information such as metadata for storage, a record of submitted compute jobs, users, groups, and associated permissions.| -|arv-git-httpd|Provides a git+http interface to Arvados-managed git repositories, with permissions and authentication based on an Arvados API token.| +|arvados-git-httpd|Provides a git+http interface to Arvados-managed git repositories, with permissions and authentication based on an Arvados API token.| |arvados-dispatch-cloud|Provide elastic computing by creating and destroying cloud based virtual machines on compute demand.| |crunch-dispatch-local|Get compute requests submitted to the API server and execute them locally.| |crunch-dispatch-slurm|Get compute requests submitted to the API server and submit them to slurm.| diff --git a/doc/images/Arvados_arch.svg b/doc/images/Arvados_arch.svg index 24f1f5a1d5..00a4e07d37 100644 --- a/doc/images/Arvados_arch.svg +++ b/doc/images/Arvados_arch.svg @@ -219,7 +219,7 @@ - arv-git-httpd + arvados-git-httpd diff --git a/doc/install/install-manual-prerequisites.html.textile.liquid b/doc/install/install-manual-prerequisites.html.textile.liquid index a9a91ab3cb..21b3871e01 100644 --- a/doc/install/install-manual-prerequisites.html.textile.liquid +++ b/doc/install/install-manual-prerequisites.html.textile.liquid @@ -58,7 +58,7 @@ table(table table-bordered table-condensed). |"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).| |\3=. *User interface*| |"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional. Depends on API server, keep-web, websockets server.| -|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arv-git-httpd).| +|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arvados-git-httpd).| |\3=. *Additional services*| |"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.| |"Shell server":install-shell-server.html |Synchronize (create/delete/configure) Unix shell accounts with Arvados users.|Optional.| diff --git a/doc/install/install-shell-server.html.textile.liquid b/doc/install/install-shell-server.html.textile.liquid index 6f158deda7..9b8ecee1b7 100644 --- a/doc/install/install-shell-server.html.textile.liquid +++ b/doc/install/install-shell-server.html.textile.liquid @@ -46,7 +46,7 @@ h2(#dependencies). Install Dependecies and SDKs h2(#config-git). Update Git Config -Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arv-git-httpd. We use the @--system@ flag so it takes effect for all current and future user accounts. It does not affect git's behavior when connecting to other git servers. +Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arvados-git-httpd. We use the @--system@ flag so it takes effect for all current and future user accounts. It does not affect git's behavior when connecting to other git servers. 
diff --git a/doc/user/cwl/rnaseq-cwl-training.html.textile.liquid b/doc/user/cwl/rnaseq-cwl-training.html.textile.liquid
new file mode 100644
index 0000000000..c733868ce7
--- /dev/null
+++ b/doc/user/cwl/rnaseq-cwl-training.html.textile.liquid
@@ -0,0 +1,14 @@
+---
+layout: default
+navsection: userguide
+title: "Getting Started with CWL"
+
+no_nav_left: true
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+notextile.