postinst.sh lets the early parts define a few hooks to control behavior:
* After it installs the core configuration files (database.yml, application.yml, and production.rb) to /etc/arvados/server, it calls setup_extra_conffiles. By default this is a noop function (in step2.sh).
-* Before it restarts nginx, it calls setup_before_nginx_restart. By default this is a noop function (in step2.sh). API server defines this to set up the internal git repository, if necessary.
* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database. API server uses db:structure:load. Workbench doesn't set this, which causes the postinst to skip all database work.
# can still be there, left over from a previous version of the API server package.
rm -f $RELEASE_PATH/config/initializers/omniauth.rb
}
-
-setup_before_nginx_restart() {
- # initialize git_internal_dir
- # usually /var/lib/arvados/internal.git (set in application.default.yml )
- if [ "$APPLICATION_READY" = "1" ]; then
- GIT_INTERNAL_DIR=$($COMMAND_PREFIX bin/rake config:dump 2>&1 | grep GitInternalDir | awk '{ print $2 }' |tr -d '"')
- if [ ! -e "$GIT_INTERNAL_DIR" ]; then
- run_and_report "Creating git_internal_dir '$GIT_INTERNAL_DIR'" \
- mkdir -p "$GIT_INTERNAL_DIR"
- run_and_report "Initializing git_internal_dir '$GIT_INTERNAL_DIR'" \
- git init --quiet --bare $GIT_INTERNAL_DIR
- else
- echo "Initializing git_internal_dir $GIT_INTERNAL_DIR: directory exists, skipped."
- fi
- run_and_report "Making sure '$GIT_INTERNAL_DIR' has the right permission" \
- chown -R "$WWW_OWNER:" "$GIT_INTERNAL_DIR"
- else
- echo "Initializing git_internal_dir... skipped."
- fi
-}
chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
- setup_before_nginx_restart
-
if [ -n "$SERVICE_MANAGER" ]; then
service_command "$SERVICE_MANAGER" restart "$WEB_SERVICE"
fi
if ! type setup_extra_conffiles >/dev/null 2>&1; then
setup_extra_conffiles() { return; }
fi
-if ! type setup_before_nginx_restart >/dev/null 2>&1; then
- setup_before_nginx_restart() { return; }
-fi
if [ -e /run/systemd/system ]; then
USING_SYSTEMD=1
arvados-dispatch-cloud
arvados-dispatch-lsf
arvados-docker-cleaner
- arvados-git-httpd
arvados-health
arvados-server
arvados-src
"Arvados cluster cloud dispatch"
package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \
"Dispatch Arvados containers to an LSF cluster"
-package_go_binary cmd/arvados-server arvados-git-httpd "$FORMAT" "$ARCH" \
- "Provide authenticated http access to Arvados-hosted git repositories"
package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
"Dispatch Crunch containers on the local system"
package_go_binary cmd/arvados-server crunch-dispatch-slurm "$FORMAT" "$ARCH" \
Restrict apiserver tests to the given file
sdk/python_test="--test-suite tests.test_keep_locator"
Restrict Python SDK tests to the given class
-services/githttpd_test="-check.vv"
+lib/dispatchcloud_test="-check.vv"
Show all log messages, even when tests pass (also works
with services/keepstore_test etc.)
ARVADOS_DEBUG=1
lib/pam
lib/service
services/api
-services/githttpd
services/dockercleaner
services/fuse
services/fuse:py3
echo -n 'nginx: '
PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
|| fatal "No nginx. Try: apt-get install nginx"
- echo -n 'gitolite: '
- which gitolite \
- || fatal "No gitolite. Try: apt-get install gitolite3"
echo -n 'npm: '
npm --version \
|| fatal "No npm. Try: wget -O- https://nodejs.org/dist/v12.22.12/node-v12.22.12-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v12.22.12-linux-x64/bin/{node,npm} /usr/local/bin/"
return 0
fi
. "$VENV3DIR/bin/activate"
- echo 'Starting API, controller, keepproxy, keep-web, githttpd, ws, and nginx ssl proxy...'
+ echo 'Starting API, controller, keepproxy, keep-web, ws, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
mkdir -p "$WORKSPACE/services/api/log"
fi
&& python3 sdk/python/tests/run_test_server.py start_keep-web \
&& checkpidfile keep-web \
&& checkhealth WebDAV \
- && python3 sdk/python/tests/run_test_server.py start_githttpd \
- && checkpidfile githttpd \
- && checkhealth GitHTTP \
&& python3 sdk/python/tests/run_test_server.py start_ws \
&& checkpidfile ws \
&& export ARVADOS_TEST_PROXY_SERVICES=1 \
. "$VENV3DIR/bin/activate" || return
cd "$WORKSPACE" \
&& python3 sdk/python/tests/run_test_server.py stop_nginx \
- && python3 sdk/python/tests/run_test_server.py stop_githttpd \
&& python3 sdk/python/tests/run_test_server.py stop_ws \
&& python3 sdk/python/tests/run_test_server.py stop_keep-web \
&& python3 sdk/python/tests/run_test_server.py stop_keep_proxy \
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-[Unit]
-Description=Arvados git server
-Documentation=https://doc.arvados.org/
-After=network.target
-AssertPathExists=/etc/arvados/config.yml
-StartLimitIntervalSec=0
-
-[Service]
-Type=notify
-EnvironmentFile=-/etc/arvados/environment
-ExecStart=/usr/bin/arvados-git-httpd
-# Set a reasonable default for the open file limit
-LimitNOFILE=65536
-Restart=always
-RestartSec=1
-RestartPreventExitStatus=2
-
-[Install]
-WantedBy=multi-user.target
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/health"
dispatchslurm "git.arvados.org/arvados.git/services/crunch-dispatch-slurm"
- "git.arvados.org/arvados.git/services/githttpd"
keepbalance "git.arvados.org/arvados.git/services/keep-balance"
keepweb "git.arvados.org/arvados.git/services/keep-web"
"git.arvados.org/arvados.git/services/keepproxy"
"dispatch-cloud": dispatchcloud.Command,
"dispatch-lsf": lsf.DispatchCommand,
"dispatch-slurm": dispatchslurm.Command,
- "git-httpd": githttpd.Command,
"health": healthCommand,
"install": install.Command,
"init": install.InitCommand,
- api/methods/pipeline_instances.html.textile.liquid
- api/methods/pipeline_templates.html.textile.liquid
- api/methods/nodes.html.textile.liquid
- - api/methods/repositories.html.textile.liquid
- api/methods/keep_disks.html.textile.liquid
- Metadata for bioinformatics (legacy):
- api/methods/humans.html.textile.liquid
- install/install-ws.html.textile.liquid
- install/install-workbench2-app.html.textile.liquid
- install/workbench.html.textile.liquid
-# - install/install-composer.html.textile.liquid
- Additional services:
- install/install-shell-server.html.textile.liquid
- install/install-webshell.html.textile.liquid
- - install/install-arv-git-httpd.html.textile.liquid
-# - Containers API (all):
-# - install/install-jobs-image.html.textile.liquid
- Containers API (cloud):
- install/crunch2-cloud/install-compute-node.html.textile.liquid
- install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid
"kind":"collection",
"uuid":"..."
}</code></pre>|
-|Git tree|@git_tree@|@"uuid"@ must be the UUID of an Arvados-hosted git repository.
-@"commit"@ must be a full 40-character commit hash.
-@"path"@, if provided, must be "/".
-At container startup, the target path will have the source tree indicated by the given commit. The @.git@ metadata directory _will not_ be available.|<pre><code>{
- "kind":"git_tree",
- "uuid":"zzzzz-s0uqq-xxxxxxxxxxxxxxx",
- "commit":"f315c59f90934cccae6381e72bba59d27ba42099"
-}
-</code></pre>|
|Temporary directory|@tmp@|@"capacity"@: capacity (in bytes) of the storage device.
@"device_type"@ (optional, default "network"): one of @{"ram", "ssd", "disk", "network"}@ indicating the acceptable level of performance. (*note: not yet implemented as of v1.5*)
At container startup, the target path will be empty. When the container finishes, the content will be discarded. This will be backed by a storage mechanism no slower than the specified type.|<pre><code>{
{% endcomment %}
-Arvados requires a public SSH key in order to securely log in to an Arvados VM instance, or to access an Arvados Git repository. The three sections below help you get started:
+Arvados requires a public SSH key in order to securely log in to an Arvados VM instance. The three sections below help you get started:
# "Getting your SSH key":#gettingkey
# "Adding your key to Arvados Workbench":#workbench
+++ /dev/null
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{% include 'notebox_begin' %}
-This tutorial assumes that you have a working Arvados repository. If you do not have a repository created, you can follow the instructions in the "Adding a new repository":{{site.baseurl}}/user/tutorials/add-new-repository.html page. We will use the *$USER/tutorial* repository created in that page as the example.
-{% include 'notebox_end' %}
|controller |yes |yes|yes ^2,4^|InternalURLs used by reverse proxy and container shell connections|
|arvados-dispatch-cloud|no |yes|no ^3^|InternalURLs only used to expose Prometheus metrics|
|arvados-dispatch-lsf|no |yes|no ^3^|InternalURLs only used to expose Prometheus metrics|
-|git-http |yes |yes|no ^2^|InternalURLs only used by reverse proxy (e.g. Nginx)|
|git-ssh |yes |no |no ||
|keepproxy |yes |yes|no ^2^|InternalURLs only used by reverse proxy (e.g. Nginx)|
|keepstore |no |yes|yes |All clients connect to InternalURLs|
|arvados-controller|✓|
|arvados-dispatch-cloud|✓|
|arvados-dispatch-lsf|✓|
-|arvados-git-httpd||
|arvados-ws|✓|
|composer||
|keepproxy|✓|
The following services also support monitoring.
* API server
-* arvados-git-httpd
* controller
* keep-balance
* keepproxy
|arvados-controller|✓|
|arvados-dispatch-cloud|✓|
|arvados-dispatch-lsf|✓|
-|arvados-git-httpd||
|arvados-ws|✓|
-|composer||
|keepproxy|✓|
|keepstore|✓|
|keep-balance|✓|
}
EOF
</pre>
-
-h3. Git repository
-
-Give @$user_uuid@ permission to commit to @$repo_uuid@ as @$repo_username@
-
-<pre>
-user_uuid=xxxxxxxchangeme
-repo_uuid=xxxxxxxchangeme
-repo_username=xxxxxxxchangeme
-
-read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
-{
-"tail_uuid":"$user_uuid",
-"head_uuid":"$repo_uuid",
-"link_class":"permission",
-"name":"can_write",
-"properties":{"username":"$repo_username"}
-}
-EOF
-</pre>
# A new user record is not set up, and not active. An inactive user cannot create or update any object, but can read Arvados objects that the user account has permission to read (such as publicly available items readable by the "anonymous" user).
# Using Workbench or the "command line":{{site.baseurl}}/admin/user-management-cli.html , the admin invokes @setup@ on the user. The setup method adds the user to the "All users" group.
- If "Users.AutoSetupNewUsers":config.html is true, this happens automatically during user creation, so in that case new users start at step (3).
-- If "Users.AutoSetupNewUsersWithRepository":config.html is true, a new git repo is created for the user.
- If "Users.AutoSetupNewUsersWithVmUUID":config.html is set, the user is given login permission to the specified shell node
# User is set up, but still not yet active. The browser presents "user agreements":#user_agreements (if any) and then invokes the user @activate@ method on the user's behalf.
# The user @activate@ method checks that all "user agreements":#user_agreements are signed. If so, or there are no user agreements, the user is activated.
table(table table-bordered table-condensed).
|_. Component|_. Description|
|api|The API server is the core of Arvados. It is backed by a Postgres database and manages information such as metadata for storage, a record of submitted compute jobs, users, groups, and associated permissions.|
-|arvados-git-httpd|Provides a git+http interface to Arvados-managed git repositories, with permissions and authentication based on an Arvados API token.|
|arvados-dispatch-cloud|Provide elastic computing by creating and destroying cloud based virtual machines on compute demand.|
|crunch-dispatch-local|Get compute requests submitted to the API server and execute them locally.|
|crunch-dispatch-slurm|Get compute requests submitted to the API server and submit them to slurm.|
Arvados-in-a-box starting
-Waiting for workbench2 websockets workbench webshell keep-web controller keepproxy api keepstore1 arv-git-httpd keepstore0 sdk vm ...
+Waiting for workbench2 websockets workbench webshell keep-web controller keepproxy api keepstore1 keepstore0 sdk vm ...
...
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install the Git server
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-# "Introduction":#introduction
-# "Install dependencies":#dependencies
-# "Create "git" user and storage directory":#create
-# "Install gitolite":#gitolite
-# "Configure gitolite":#config-gitolite
-# "Configure git synchronization":#sync
-# "Update config.yml":#update-config
-# "Update nginx configuration":#update-nginx
-# "Install arvados-git-httpd package":#install-packages
-# "Restart the API server and controller":#restart-api
-# "Confirm working installation":#confirm-working
-
-h2(#introduction). Introduction
-
-Arvados support for git repository management enables using Arvados permissions to control access to git repositories. Users can create their own private and public git repositories and share them with others.
-
-The git hosting setup involves three components.
-* The "arvados-git-sync.rb" script polls the API server for the current list of repositories, creates bare repositories, and updates the local permission cache used by gitolite.
-* Gitolite provides SSH access. Users authenticate by SSH keys.
-* arvados-git-http provides HTTPS access. Users authenticate by Arvados tokens.
-
-Git services must be installed on the same host as the Arvados Rails API server.
-
-h2(#dependencies). Install dependencies
-
-h3. Alma/CentOS/Red Hat/Rocky
-
-<notextile>
-<pre><code># <span class="userinput">dnf install git perl-Data-Dumper openssh-server</span>
-</code></pre>
-</notextile>
-
-h3. Debian and Ubuntu
-
-<notextile>
-<pre><code># <span class="userinput">apt-get --no-install-recommends install git openssh-server</span>
-</code></pre>
-</notextile>
-
-h2(#create). Create "git" user and storage directory
-
-Gitolite and some additional scripts will be installed in @/var/lib/arvados/git@, which means hosted repository data will be stored in @/var/lib/arvados/git/repositories@. If you choose to install gitolite in a different location, make sure to update the @git_repositories_dir@ entry in your API server's @application.yml@ file accordingly: for example, if you install gitolite at @/data/gitolite@ then your @git_repositories_dir@ will be @/data/gitolite/repositories@.
-
-A new UNIX account called "git" will own the files. This makes git URLs look familiar to users (<code>git@[...]:username/reponame.git</code>).
-
-On Debian- or Red Hat-based systems:
-
-<notextile>
-<pre><code>gitserver:~$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git</span>
-gitserver:~$ <span class="userinput">sudo useradd --comment git --home-dir /var/lib/arvados/git git</span>
-gitserver:~$ <span class="userinput">sudo chown -R git:git ~git</span>
-</code></pre>
-</notextile>
-
-The git user needs its own SSH key. (It must be able to run <code>ssh git@localhost</code> from scripts.)
-
-<notextile>
-<pre><code>gitserver:~$ <span class="userinput">sudo -u git -i bash</span>
-git@gitserver:~$ <span class="userinput">ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa</span>
-git@gitserver:~$ <span class="userinput">cp .ssh/id_rsa.pub .ssh/authorized_keys</span>
-git@gitserver:~$ <span class="userinput">ssh -o stricthostkeychecking=no localhost cat .ssh/id_rsa.pub</span>
-Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts.
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7aBIDAAgMQN16Pg6eHmvc+D+6TljwCGr4YGUBphSdVb25UyBCeAEgzqRiqy0IjQR2BLtSirXr+1SJAcQfBgI/jwR7FG+YIzJ4ND9JFEfcpq20FvWnMMQ6XD3y3xrZ1/h/RdBNwy4QCqjiXuxDpDB7VNP9/oeAzoATPZGhqjPfNS+RRVEQpC6BzZdsR+S838E53URguBOf9yrPwdHvosZn7VC0akeWQerHqaBIpSfDMtaM4+9s1Gdsz0iP85rtj/6U/K/XOuv2CZsuVZZ52nu3soHnEX2nx2IaXMS3L8Z+lfOXB2T6EaJgXF7Z9ME5K1tx9TSNTRcYCiKztXLNLSbp git@gitserver
-git@gitserver:~$ <span class="userinput">rm .ssh/authorized_keys</span>
-</code></pre>
-</notextile>
-
-h2(#gitolite). Install gitolite
-
-Check "https://github.com/sitaramc/gitolite/tags":https://github.com/sitaramc/gitolite/tags for the latest stable version. This guide was tested with @v3.6.11@. _Versions below 3.0 are missing some features needed by Arvados, and should not be used._
-
-Download and install the version you selected.
-
-<notextile>
-<pre><code>$ <span class="userinput">sudo -u git -i bash</span>
-git@gitserver:~$ <span class="userinput">echo 'PATH=$HOME/bin:$PATH' >.profile</span>
-git@gitserver:~$ <span class="userinput">. .profile</span>
-git@gitserver:~$ <span class="userinput">git clone --branch <b>v3.6.11</b> https://github.com/sitaramc/gitolite</span>
-...
-Note: checking out '5d24ae666bfd2fa9093d67c840eb8d686992083f'.
-...
-git@gitserver:~$ <span class="userinput">mkdir bin</span>
-git@gitserver:~$ <span class="userinput">gitolite/install -ln ~git/bin</span>
-git@gitserver:~$ <span class="userinput">bin/gitolite setup -pk .ssh/id_rsa.pub</span>
-Initialized empty Git repository in /var/lib/arvados/git/repositories/gitolite-admin.git/
-Initialized empty Git repository in /var/lib/arvados/git/repositories/testing.git/
-WARNING: /var/lib/arvados/git/.ssh/authorized_keys missing; creating a new one
- (this is normal on a brand new install)
-</code></pre>
-</notextile>
-
-_If this didn't go well, more detail about installing gitolite, and information about how it works, can be found on the "gitolite home page":http://gitolite.com/._
-
-Clone the gitolite-admin repository. The arvados-git-sync.rb script works by editing the files in this working directory and pushing them to gitolite. Here we make sure "git push" won't produce any errors or warnings.
-
-<notextile>
-<pre><code>git@gitserver:~$ <span class="userinput">git clone git@localhost:gitolite-admin</span>
-Cloning into 'gitolite-admin'...
-remote: Counting objects: 6, done.
-remote: Compressing objects: 100% (4/4), done.
-remote: Total 6 (delta 0), reused 0 (delta 0)
-Receiving objects: 100% (6/6), done.
-Checking connectivity... done.
-git@gitserver:~$ <span class="userinput">cd gitolite-admin</span>
-git@gitserver:~/gitolite-admin$ <span class="userinput">git config user.email arvados</span>
-git@gitserver:~/gitolite-admin$ <span class="userinput">git config user.name arvados</span>
-git@gitserver:~/gitolite-admin$ <span class="userinput">git config push.default simple</span>
-git@gitserver:~/gitolite-admin$ <span class="userinput">git push</span>
-Everything up-to-date
-</code></pre>
-</notextile>
-
-h2(#config-gitolite). Configure gitolite
-
-Configure gitolite to look up a repository name like @username/reponame.git@ and find the appropriate bare repository storage directory.
-
-Add the following lines to the top of @~git/.gitolite.rc@:
-
-<notextile>
-<pre><code><span class="userinput">my $repo_aliases;
-my $aliases_src = "$ENV{HOME}/.gitolite/arvadosaliases.pl";
-if ($ENV{HOME} && (-e $aliases_src)) {
- $repo_aliases = do $aliases_src;
-}
-$repo_aliases ||= {};
-</span></code></pre>
-</notextile>
-
-Add the following lines inside the section that begins @%RC = (@:
-
-<notextile>
-<pre><code><span class="userinput"> REPO_ALIASES => $repo_aliases,
-</span></code></pre>
-</notextile>
-
-Inside that section, adjust the 'UMASK' setting to @022@, to ensure the API server has permission to read repositories:
-
-<notextile>
-<pre><code> UMASK => <span class="userinput">022</span>,
-</code></pre>
-</notextile>
-
-Uncomment the 'Alias' line in the section that begins @ENABLE => [@:
-
-<notextile>
-<pre><code><span class="userinput"> # access a repo by another (possibly legacy) name
- 'Alias',
-</span></code></pre>
-</notextile>
-
-h2(#sync). Configure git synchronization
-
-Create a configuration file @/var/www/arvados-api/current/config/arvados-clients.yml@ using the following template, filling in the appropriate values for your system.
-* For @arvados_api_token@, use @SystemRootToken@
-* For @gitolite_arvados_git_user_key@, provide the public key you generated above, i.e., the contents of @~git/.ssh/id_rsa.pub@.
-
-<notextile>
-<pre><code>production:
- gitolite_url: /var/lib/arvados/git/repositories/gitolite-admin.git
- gitolite_tmp: /var/lib/arvados/git
- arvados_api_host: <span class="userinput">ClusterID.example.com</span>
- arvados_api_token: "<span class="userinput">zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</span>"
- arvados_api_host_insecure: <span class="userinput">false</span>
- gitolite_arvados_git_user_key: "<span class="userinput">ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7aBIDAAgMQN16Pg6eHmvc+D+6TljwCGr4YGUBphSdVb25UyBCeAEgzqRiqy0IjQR2BLtSirXr+1SJAcQfBgI/jwR7FG+YIzJ4ND9JFEfcpq20FvWnMMQ6XD3y3xrZ1/h/RdBNwy4QCqjiXuxDpDB7VNP9/oeAzoATPZGhqjPfNS+RRVEQpC6BzZdsR+S838E53URguBOf9yrPwdHvosZn7VC0akeWQerHqaBIpSfDMtaM4+9s1Gdsz0iP85rtj/6U/K/XOuv2CZsuVZZ52nu3soHnEX2nx2IaXMS3L8Z+lfOXB2T6EaJgXF7Z9ME5K1tx9TSNTRcYCiKztXLNLSbp git@gitserver</span>"
-</code></pre>
-</notextile>
-
-<pre>
-$ sudo chown git:git /var/www/arvados-api/current/config/arvados-clients.yml
-$ sudo chmod og-rwx /var/www/arvados-api/current/config/arvados-clients.yml
-</pre>
-
-h3. Test configuration
-
-notextile. <pre><code>$ <span class="userinput">sudo -u git -i bash -c 'cd /var/www/arvados-api/current && bin/bundle exec script/arvados-git-sync.rb production'</span></code></pre>
-
-h3. Enable the synchronization script
-
-The API server package includes a script that retrieves the current set of repository names and permissions from the API, writes them to @arvadosaliases.pl@ in a format usable by gitolite, and triggers gitolite hooks which create new empty repositories if needed. This script should run every 2 to 5 minutes.
-
-Create @/etc/cron.d/arvados-git-sync@ with the following content:
-
-<notextile>
-<pre><code><span class="userinput">*/5 * * * * git cd /var/www/arvados-api/current && bin/bundle exec script/arvados-git-sync.rb production</span>
-</code></pre>
-</notextile>
-
-h2(#update-config). Update config.yml
-
-Edit the cluster config at @config.yml@ .
-
-<notextile>
-<pre><code> Services:
- GitSSH:
- ExternalURL: "<span class="userinput">ssh://git@git.ClusterID.example.com</span>"
- GitHTTP:
- ExternalURL: <span class="userinput">https://git.ClusterID.example.com/</span>
- InternalURLs:
- "http://localhost:9001": {}
- Git:
- GitCommand: <span class="userinput">/var/lib/arvados/git/gitolite/src/gitolite-shell</span>
- GitoliteHome: <span class="userinput">/var/lib/arvados/git</span>
- Repositories: <span class="userinput">/var/lib/arvados/git/repositories</span>
-</code></pre>
-</notextile>
-
-h2(#update-nginx). Update nginx configuration
-
-Use a text editor to create a new file @/etc/nginx/conf.d/arvados-git.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
-
-<notextile>
-<pre><code>upstream arvados-git-httpd {
- server 127.0.0.1:<span class="userinput">9001</span>;
-}
-server {
- listen 443 ssl;
- server_name git.<span class="userinput">ClusterID.example.com</span>;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
- # The server needs to accept potentially large refpacks from push clients.
- client_max_body_size 128m;
-
- location / {
- proxy_pass http://arvados-git-httpd;
- }
-}
-</code></pre>
-</notextile>
-
-h2(#install-packages). Install the arvados-git-httpd package
-
-The arvados-git-httpd package provides HTTP access, using Arvados authentication tokens instead of passwords. It must be installed on the system where your git repositories are stored.
-
-h3. Alma/CentOS/Red Hat/Rocky
-
-<notextile>
-<pre><code># <span class="userinput">dnf install arvados-git-httpd</span>
-</code></pre>
-</notextile>
-
-h3. Debian and Ubuntu
-
-<notextile>
-<pre><code># <span class="userinput">apt-get --no-install-recommends install arvados-git-httpd</span>
-</code></pre>
-</notextile>
-
-h2(#restart-api). Restart the API server and controller
-
-After adding Workbench to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
-
-<notextile>
-<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
-</code></pre>
-</notextile>
-
-h2(#confirm-working). Confirm working installation
-
-Create 'testrepo' in the Arvados database.
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv --format=uuid repository create --repository '{"name":"myusername/testrepo"}'</span>
-</code></pre></notextile>
-
-The arvados-git-sync cron job will notice the new repository record and create a repository on disk. Because it is on a timer (default 5 minutes) you may have to wait a minute or two for it to show up.
-
-h3. SSH
-
-Before you do this, go to Workbench and choose *SSH Keys* from the menu, and upload your public key. Arvados uses the public key to identify you when you access the git repo.
-
-<notextile>
-<pre><code>~$ <span class="userinput">git clone git@git.ClusterID.example.com:username/testrepo.git</span>
-</code></pre>
-</notextile>
-
-h3. HTTP
-
-Set up git credential helpers as described in "install shell server":install-shell-server.html#config-git for the git command to use your API token instead of prompting you for a username and password.
-
-<notextile>
-<pre><code>~$ <span class="userinput">git clone https://git.ClusterID.example.com/username/testrepo.git</span>
-</code></pre>
-</notextile>
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install Composer
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-Arvados Composer is a web-based javascript application for building Common Workflow Languge (CWL) Workflows.
-
-# "Install dependencies":#dependencies
-# "Update config.yml":#update-config
-# "Update Nginx configuration":#update-nginx
-# "Install arvados-composer":#install-packages
-# "Restart the API server and controller":#restart-api
-# "Confirm working installation":#confirm-working
-
-h2(#dependencies). Install dependencies
-
-In addition to Arvados core services, Composer requires "Arvados hosted git repositories":install-arv-git-httpd.html which are used for storing workflow files.
-
-h2(#configure). Update config.yml
-
-Edit @config.yml@ and set @Services.Composer.ExternalURL@ to the location from which it is served:
-
-<notextile>
-<pre><code> Services:
- Composer:
- ExternalURL: <span class="userinput">https://workbench.CusterID.example.com/composer</span></code></pre>
-</notextile>
-
-h2(#update-nginx). Update nginx configuration
-
-Composer may be served from the same host as Workbench. Composer communicates directly with the Arvados API server. It does not require its own backend and should be served as a static file.
-
-Add the following @location@ sections to @/etc/nginx/conf.d/arvados-workbench.conf@ .
-
-<notextile>
-<pre><code>server {
- [...]
-
- location /composer {
- root /var/www/arvados-composer;
- index index.html;
- }
-
- location /composer/composer.yml {
- return 200 '{ "API_HOST": "<span class="userinput">ClusterID.example.com</span>" }';
- }
-}
-</code></pre>
-</notextile>
-
-{% assign arvados_component = 'arvados-composer' %}
-
-{% include 'install_packages' %}
-
-{% include 'restart_api' %}
-
-h2(#confirm-working). Confirm working installation
-
-Visit @https://workbench.ClusterID.example.com/composer@ in a browser. You should be able to log in using the login method you configured previously.
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install arvados/jobs image
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-h2. Create a project for Docker images
-
-Here we create a default project for the standard Arvados Docker images, and give all users read access to it. The project is owned by the system user.
-
-<notextile>
-<pre><code>~$ <span class="userinput">uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)</span>
-~$ <span class="userinput">project_uuid=$(arv --format=uuid group create --group '{"owner_uuid":"'$uuid_prefix'-tpzed-000000000000000", "group_class":"project", "name":"Arvados Standard Docker Images"}')</span>
-~$ <span class="userinput">echo "Arvados project uuid is '$project_uuid'"</span>
-~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
-<span class="userinput">{
- "tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
- "head_uuid":"$project_uuid",
- "link_class":"permission",
- "name":"can_read"
-}
-EOF</span>
-</code></pre></notextile>
-
-h2. Import the arvados/jobs docker image
-
-In order to start workflows from workbench, there needs to be Docker image @arvados/jobs@ tagged with the version of Arvados you are installing. The following command downloads the latest arvados/jobs image from Docker Hub, loads it into Keep. In this example @$project_uuid@ should be the UUID of the "Arvados Standard Docker Images" project.
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv-keepdocker --pull arvados/jobs latest --project-uuid $project_uuid</span>
-</code></pre></notextile>
-
-If the image needs to be downloaded from Docker Hub, the command can take a few minutes to complete, depending on available network bandwidth.
|\3=. *Additional services*|
|"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
|"Shell server":install-shell-server.html |Grant Arvados users access to Unix shell accounts on dedicated shell nodes.|Optional.|
-|"Git server":install-arv-git-httpd.html |Arvados-hosted git repositories, with Arvados-token based authentication.|Optional|
|\3=. *Crunch (running containers)*|
|"arvados-dispatch-cloud":crunch2-cloud/install-dispatch-cloud.html |Run analysis workflows on cloud by allocating and freeing cloud VM instances on demand.|Optional|
|"crunch-dispatch-slurm":crunch2-slurm/install-dispatch.html |Run analysis workflows distributed across a Slurm cluster.|Optional|
<div class="offset1">
table(table table-bordered table-condensed).
|_. Function|_. Number of nodes|_. Recommended specs|
-|PostgreSQL database, Arvados API server, Arvados controller, Git, Websockets, Container dispatcher|1|16+ GiB RAM, 4+ cores, fast disk for database|
+|PostgreSQL database, Arvados API server, Arvados controller, Websockets, Container dispatcher|1|16+ GiB RAM, 4+ cores, fast disk for database|
|Workbench, Keepproxy, Keep-web, Keep-balance|1|8 GiB RAM, 2+ cores|
|Keepstore servers ^1^|2+|4 GiB RAM|
|Compute worker nodes ^1^|0+ |Depends on workload; scaled dynamically in the cloud|
table(table table-bordered table-condensed).
|_. Function|_. DNS name|
|Arvados API|@ClusterID.example.com@|
-|Arvados Git server|git.@ClusterID.example.com@|
|Arvados Webshell|webshell.@ClusterID.example.com@|
|Arvados Websockets endpoint|ws.@ClusterID.example.com@|
|Arvados Workbench|workbench.@ClusterID.example.com@|
# "Introduction":#introduction
# "Install Dependencies and SDKs":#dependencies
# "Install git and curl":#install-packages
-# "Update Git Config":#config-git
# "Create record for VM":#vm-record
# "Install arvados-login-sync":#arvados-login-sync
# "Confirm working installation":#confirm-working
{% include 'install_packages' %}
-h2(#config-git). Update Git Config
-
-Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arvados-git-httpd. We use the @--system@ flag so it takes effect for all current and future user accounts. It does not affect git's behavior when connecting to other git servers.
-
-<notextile>
-<pre>
-<code># <span class="userinput">git config --system 'credential.https://git.<b>ClusterID.example.com</b>/.username' none</span></code>
-<code># <span class="userinput">git config --system 'credential.https://git.<b>ClusterID.example.com</b>/.helper' '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'</span></code>
-</pre>
-</notextile>
-
h2(#vm-record). Create record for VM
As an admin, create an Arvados virtual_machine object representing this shell server. This will return a uuid.
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: Adding a new Arvados git repository
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-Arvados supports managing git repositories. You can access these repositories using your Arvados credentials and share them with other Arvados users.
-
-{% include 'tutorial_expectations' %}
-
-h2. Setting up Git
-
-Before you start using Git and arvados repositories, you should do some basic configuration (you only need to do this the first time):
-
-<notextile>
-<pre><code>~$ <span class="userinput">git config --global user.name "Your Name"</span>
-~$ <span class="userinput">git config --global user.email $USER@example.com</span></code></pre>
-</notextile>
-
-h2. Add "tutorial" repository
-
-On the Arvados Workbench, click on the dropdown menu icon <i class="fa fa-lg fa-user"></i> (Account Management) in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Repositories*.
-
-In the *Repositories* page, you will see the <span class="btn btn-sm btn-primary">+ NEW REPOSITORY</span> button.
-
-!{width: 100%;}{{ site.baseurl }}/images/repositories-panel.png!
-
-Click the <span class="btn btn-sm btn-primary">+ NEW REPOSITORY</span> button to open the popup to add a new Arvados repository. You will see a text box where you can enter the name of the repository. Enter *tutorial* in this text box and click on *Create*.
-
-{% include 'notebox_begin' %}
-The name you enter here must begin with a letter and can only contain alphanumeric characters.
-{% include 'notebox_end' %}
-
-!{width: 100%;}{{ site.baseurl }}/images/add-new-repository.png!
-
-This will create a new repository with the name @$USER/tutorial@. It can be accessed using the URL <notextile><code>https://git.{{ site.arvados_api_host }}/$USER/tutorial.git</code></notextile> or <notextile><code>git@git.{{ site.arvados_api_host }}:$USER/tutorial.git</code></notextile>
-
-Back in the *Repositories* page, you should see the @$USER/tutorial@ repository listed in the name column with these URLs.
-
-!{display: block;margin-left: 25px;margin-right: auto;}{{ site.baseurl }}/images/added-new-repository.png!
-
-You are now ready to use this *tutorial* repository to run your crunch scripts.
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: Working with an Arvados git repository
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This tutorial describes how to work with an Arvados-managed git repository. Working with an Arvados git repository is very similar to working with other public git repositories.
-
-{% include 'tutorial_expectations' %}
-
-{% include 'tutorial_git_repo_expectations' %}
-
-h2. Cloning a git repository
-
-Before you start using Git, you should do some basic configuration (you only need to do this the first time):
-
-<notextile>
-<pre><code>~$ <span class="userinput">git config --global user.name "Your Name"</span>
-~$ <span class="userinput">git config --global user.email $USER@example.com</span></code></pre>
-</notextile>
-
-On the Arvados Workbench, click on the dropdown menu icon <i class="fa fa-lg fa-user"></i> in the upper right corner of the top navigation menu to access the Account Management menu, and click on the menu item *Repositories*. In the *Repositories* page, you should see the @$USER/tutorial@ repository listed in the *name* column. Next to *name* is the column *URL*. Copy the *URL* value associated with your repository. This should look like <notextile><code>https://git.{{ site.arvados_api_host }}/$USER/tutorial.git</code></notextile>. Alternatively, you can use <notextile><code>git@git.{{ site.arvados_api_host }}:$USER/tutorial.git</code></notextile>
-
-Next, on the Arvados virtual machine, clone your Git repository:
-
-<notextile>
-<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
-~$ <span class="userinput">git clone https://git.{{ site.arvados_api_host }}/$USER/tutorial.git</span>
-Cloning into 'tutorial'...</code></pre>
-</notextile>
-
-This will create a Git repository in the directory called @tutorial@ in your home directory. Say yes when prompted to continue with connection.
-Ignore any warning that you are cloning an empty repository.
-
-*Note:* If you are prompted for username and password when you try to git clone using this command, you may first need to update your git configuration. Execute the following commands to update your git configuration.
-
-<notextile>
-<pre>
-<code>~$ <span class="userinput">git config 'credential.https://git.{{ site.arvados_api_host }}/.username' none</span></code>
-<code>~$ <span class="userinput">git config 'credential.https://git.{{ site.arvados_api_host }}/.helper' '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'</span></code>
-</pre>
-</notextile>
-
-h2. Creating a git branch in an Arvados repository
-
-Create a git branch named *tutorial_branch* in the *tutorial* Arvados git repository.
-
-<notextile>
-<pre><code>~$ <span class="userinput">cd tutorial</span>
-~/tutorial$ <span class="userinput">git checkout -b tutorial_branch</span>
-</code></pre>
-</notextile>
-
-h2. Adding scripts to an Arvados repository
-
-A git repository is a good place to store the CWL workflows that you run on Arvados.
-
-First, create a simple CWL CommandLineTool:
-
-notextile. <pre>~/tutorials$ <code class="userinput">nano hello.cwl</code></pre>
-
-<notextile> {% code tutorial_hello_cwl as yaml %} </notextile>
-
-Next, add the file to the git repository. This tells @git@ that the file should be included on the next commit.
-
-notextile. <pre><code>~/tutorial$ <span class="userinput">git add hello.cwl</span></code></pre>
-
-Next, commit your changes. All staged changes are recorded into the local git repository:
-
-<notextile>
-<pre><code>~/tutorial$ <span class="userinput">git commit -m "my first script"</span>
-</code></pre>
-</notextile>
-
-Finally, upload your changes to the remote repository:
-
-<notextile>
-<pre><code>~/tutorial/crunch_scripts$ <span class="userinput">git push origin tutorial_branch</span>
-</code></pre>
-</notextile>
-
-The same steps can be used to add any of your custom bash, R, or python scripts to an Arvados repository.
{"KEEPWEB", super.cluster.Services.WebDAV},
{"KEEPWEBDL", super.cluster.Services.WebDAVDownload},
{"KEEPPROXY", super.cluster.Services.Keepproxy},
- {"GIT", super.cluster.Services.GitHTTP},
{"HEALTH", super.cluster.Services.Health},
{"WORKBENCH1", super.cluster.Services.Workbench1},
{"WORKBENCH2", super.cluster.Services.Workbench2},
runNginx{},
railsDatabase{},
runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{railsDatabase{}}},
- runServiceCommand{name: "git-httpd", svc: super.cluster.Services.GitHTTP},
runServiceCommand{name: "health", svc: super.cluster.Services.Health},
runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
for _, svc := range []*arvados.Service{
&super.cluster.Services.Controller,
&super.cluster.Services.DispatchCloud,
- &super.cluster.Services.GitHTTP,
&super.cluster.Services.Health,
&super.cluster.Services.Keepproxy,
&super.cluster.Services.Keepstore,
}
host := net.JoinHostPort(defaultExtHost, port)
if svc == &super.cluster.Services.Controller ||
- svc == &super.cluster.Services.GitHTTP ||
svc == &super.cluster.Services.Health ||
svc == &super.cluster.Services.Keepproxy ||
svc == &super.cluster.Services.WebDAV ||
Keepbalance:
InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
- GitHTTP:
- InternalURLs: {SAMPLE: {ListenURL: ""}}
- ExternalURL: ""
- GitSSH:
- InternalURLs: {SAMPLE: {ListenURL: ""}}
- ExternalURL: ""
DispatchCloud:
InternalURLs: {SAMPLE: {ListenURL: ""}}
ExternalURL: ""
# AutoSetupUsernameBlacklist is a list of usernames to be blacklisted for auto setup.
AutoSetupNewUsers: false
AutoSetupNewUsersWithVmUUID: ""
- AutoSetupNewUsersWithRepository: false
AutoSetupUsernameBlacklist:
arvados: {}
git: {}
# production use.
TrustPrivateNetworks: false
- Git:
- # Path to git or gitolite-shell executable. Each authenticated
- # request will execute this program with the single argument "http-backend"
- GitCommand: /usr/bin/git
-
- # Path to Gitolite's home directory. If a non-empty path is given,
- # the CGI environment will be set up to support the use of
- # gitolite-shell as a GitCommand: for example, if GitoliteHome is
- # "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
- # PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
- GitoliteHome: ""
-
- # Git repositories must be readable by api server, or you won't be
- # able to submit crunch jobs. To pass the test suites, put a clone
- # of the arvados tree in {git_repositories_dir}/arvados.git or
- # {git_repositories_dir}/arvados/.git
- Repositories: /var/lib/arvados/git/repositories
-
TLS:
# Use "file:///var/lib/acme/live/example.com/cert" and
# ".../privkey" to load externally managed certificates.
# 'false' -- disable the Jobs API despite presence of existing records.
Enable: 'auto'
- # Git repositories must be readable by api server, or you won't be
- # able to submit crunch jobs. To pass the test suites, put a clone
- # of the arvados tree in {git_repositories_dir}/arvados.git or
- # {git_repositories_dir}/arvados/.git
- GitInternalDir: /var/lib/arvados/internal.git
-
CloudVMs:
# Enable the cloud scheduler.
Enable: false
return nil
}
-const defaultGitHttpdConfigPath = "/etc/arvados/git-httpd/git-httpd.yml"
-
-type oldGitHttpdConfig struct {
- Client *arvados.Client
- Listen *string
- GitCommand *string
- GitoliteHome *string
- RepoRoot *string
- ManagementToken *string
-}
-
-func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error {
- if ldr.GitHttpdPath == "" {
- return nil
- }
- var oc oldGitHttpdConfig
- err := ldr.loadOldConfigHelper("arvados-git-httpd", ldr.GitHttpdPath, &oc)
- if os.IsNotExist(err) && ldr.GitHttpdPath == defaultGitHttpdConfigPath {
- return nil
- } else if err != nil {
- return err
- }
-
- cluster, err := cfg.GetCluster("")
- if err != nil {
- return err
- }
-
- loadOldClientConfig(cluster, oc.Client)
-
- if oc.Listen != nil {
- cluster.Services.GitHTTP.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
- }
- if oc.ManagementToken != nil {
- cluster.ManagementToken = *oc.ManagementToken
- }
- if oc.GitCommand != nil {
- cluster.Git.GitCommand = *oc.GitCommand
- }
- if oc.GitoliteHome != nil {
- cluster.Git.GitoliteHome = *oc.GitoliteHome
- }
- if oc.RepoRoot != nil {
- cluster.Git.Repositories = *oc.RepoRoot
- }
-
- cfg.Clusters[cluster.ClusterID] = *cluster
- return nil
-}
-
const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml"
type oldKeepBalanceConfig struct {
`, debugLog, param)
}
-func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
- content := []byte(`
-{
- "Client": {
- "Scheme": "",
- "APIHost": "example.com",
- "AuthToken": "abcdefg",
- },
- "Listen": ":9000",
- "GitCommand": "/test/git",
- "GitoliteHome": "/test/gitolite",
- "RepoRoot": "/test/reporoot",
- "ManagementToken": "xyzzy"
-}
-`)
- f := "-legacy-git-httpd-config"
- cluster, err := testLoadLegacyConfig(content, f, c)
-
- c.Assert(err, check.IsNil)
- c.Assert(cluster, check.NotNil)
- c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
- c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
- c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
- c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
- c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
- c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
- c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
-}
-
-// Tests fix for https://dev.arvados.org/issues/15642
-func (s *LoadSuite) TestLegacyArvGitHttpdConfigDoesntDisableMissingItems(c *check.C) {
- content := []byte(`
-{
- "Client": {
- "Scheme": "",
- "APIHost": "example.com",
- "AuthToken": "abcdefg",
- }
-}
-`)
- cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
- c.Assert(err, check.IsNil)
- // The resulting ManagementToken should be the one set up on the test server.
- c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
-}
-
func (s *LoadSuite) TestLegacyKeepBalanceConfig(c *check.C) {
f := "-legacy-keepbalance-config"
content := []byte(fmtKeepBalanceConfig(""))
// exists.
var whitelist = map[string]bool{
// | sort -t'"' -k2,2
- "API": true,
- "API.AsyncPermissionsUpdateInterval": false,
- "API.DisabledAPIs": false,
- "API.FreezeProjectRequiresDescription": true,
- "API.FreezeProjectRequiresProperties": true,
- "API.FreezeProjectRequiresProperties.*": true,
- "API.KeepServiceRequestTimeout": false,
- "API.LockBeforeUpdate": false,
- "API.LogCreateRequestFraction": false,
- "API.MaxConcurrentRailsRequests": false,
- "API.MaxConcurrentRequests": false,
- "API.MaxGatewayTunnels": false,
- "API.MaxIndexDatabaseRead": false,
- "API.MaxItemsPerResponse": true,
- "API.MaxKeepBlobBuffers": false,
- "API.MaxQueuedRequests": false,
- "API.MaxQueueTimeForLockRequests": false,
- "API.MaxRequestAmplification": false,
- "API.MaxRequestSize": true,
- "API.MaxTokenLifetime": false,
- "API.RequestTimeout": true,
- "API.SendTimeout": true,
- "API.UnfreezeProjectRequiresAdmin": true,
- "API.VocabularyPath": false,
- "API.WebsocketClientEventQueue": false,
- "API.WebsocketServerEventQueue": false,
- "AuditLogs": false,
- "AuditLogs.MaxAge": false,
- "AuditLogs.MaxDeleteBatch": false,
- "AuditLogs.UnloggedAttributes": false,
- "ClusterID": true,
- "Collections": true,
- "Collections.BalanceCollectionBatch": false,
- "Collections.BalanceCollectionBuffers": false,
- "Collections.BalancePeriod": false,
- "Collections.BalancePullLimit": false,
- "Collections.BalanceTimeout": false,
- "Collections.BalanceTrashLimit": false,
- "Collections.BalanceUpdateLimit": false,
- "Collections.BlobDeleteConcurrency": false,
- "Collections.BlobMissingReport": false,
- "Collections.BlobReplicateConcurrency": false,
- "Collections.BlobSigning": true,
- "Collections.BlobSigningKey": false,
- "Collections.BlobSigningTTL": true,
- "Collections.BlobTrash": false,
- "Collections.BlobTrashCheckInterval": false,
- "Collections.BlobTrashConcurrency": false,
- "Collections.BlobTrashLifetime": false,
- "Collections.CollectionVersioning": true,
- "Collections.DefaultReplication": true,
- "Collections.DefaultTrashLifetime": true,
- "Collections.ForwardSlashNameSubstitution": true,
- "Collections.KeepproxyPermission": false,
- "Collections.ManagedProperties": true,
- "Collections.ManagedProperties.*": true,
- "Collections.ManagedProperties.*.*": true,
- "Collections.PreserveVersionIfIdle": true,
- "Collections.S3FolderObjects": true,
- "Collections.TrashSweepInterval": false,
- "Collections.TrustAllContent": true,
- "Collections.WebDAVCache": false,
- "Collections.WebDAVLogEvents": false,
- "Collections.WebDAVOutputBuffer": false,
- "Collections.WebDAVPermission": false,
- "Containers": true,
- "Containers.AlwaysUsePreemptibleInstances": true,
- "Containers.CloudVMs": false,
- "Containers.CrunchRunArgumentsList": false,
- "Containers.CrunchRunCommand": false,
- "Containers.DefaultKeepCacheRAM": true,
- "Containers.DispatchPrivateKey": false,
- "Containers.JobsAPI": true,
- "Containers.JobsAPI.Enable": true,
- "Containers.JobsAPI.GitInternalDir": false,
- "Containers.LocalKeepBlobBuffersPerVCPU": false,
- "Containers.LocalKeepLogsToContainerLog": false,
- "Containers.Logging": false,
- "Containers.LogReuseDecisions": false,
- "Containers.LSF": false,
- "Containers.MaxDispatchAttempts": false,
- "Containers.MaximumPriceFactor": true,
- "Containers.MaxRetryAttempts": true,
- "Containers.MinRetryPeriod": true,
- "Containers.PreemptiblePriceFactor": false,
- "Containers.ReserveExtraRAM": true,
- "Containers.RuntimeEngine": true,
- "Containers.ShellAccess": true,
- "Containers.ShellAccess.Admin": true,
- "Containers.ShellAccess.User": true,
- "Containers.SLURM": false,
- "Containers.StaleLockTimeout": false,
- "Containers.SupportedDockerImageFormats": true,
- "Containers.SupportedDockerImageFormats.*": true,
- "Git": false,
- "InstanceTypes": true,
- "InstanceTypes.*": true,
- "InstanceTypes.*.*": true,
- "InstanceTypes.*.*.*": true,
- "Login": true,
- "Login.Google": true,
- "Login.Google.AlternateEmailAddresses": false,
+ "API": true,
+ "API.AsyncPermissionsUpdateInterval": false,
+ "API.DisabledAPIs": false,
+ "API.FreezeProjectRequiresDescription": true,
+ "API.FreezeProjectRequiresProperties": true,
+ "API.FreezeProjectRequiresProperties.*": true,
+ "API.KeepServiceRequestTimeout": false,
+ "API.LockBeforeUpdate": false,
+ "API.LogCreateRequestFraction": false,
+ "API.MaxConcurrentRailsRequests": false,
+ "API.MaxConcurrentRequests": false,
+ "API.MaxGatewayTunnels": false,
+ "API.MaxIndexDatabaseRead": false,
+ "API.MaxItemsPerResponse": true,
+ "API.MaxKeepBlobBuffers": false,
+ "API.MaxQueuedRequests": false,
+ "API.MaxQueueTimeForLockRequests": false,
+ "API.MaxRequestAmplification": false,
+ "API.MaxRequestSize": true,
+ "API.MaxTokenLifetime": false,
+ "API.RequestTimeout": true,
+ "API.SendTimeout": true,
+ "API.UnfreezeProjectRequiresAdmin": true,
+ "API.VocabularyPath": false,
+ "API.WebsocketClientEventQueue": false,
+ "API.WebsocketServerEventQueue": false,
+ "AuditLogs": false,
+ "AuditLogs.MaxAge": false,
+ "AuditLogs.MaxDeleteBatch": false,
+ "AuditLogs.UnloggedAttributes": false,
+ "ClusterID": true,
+ "Collections": true,
+ "Collections.BalanceCollectionBatch": false,
+ "Collections.BalanceCollectionBuffers": false,
+ "Collections.BalancePeriod": false,
+ "Collections.BalancePullLimit": false,
+ "Collections.BalanceTimeout": false,
+ "Collections.BalanceTrashLimit": false,
+ "Collections.BalanceUpdateLimit": false,
+ "Collections.BlobDeleteConcurrency": false,
+ "Collections.BlobMissingReport": false,
+ "Collections.BlobReplicateConcurrency": false,
+ "Collections.BlobSigning": true,
+ "Collections.BlobSigningKey": false,
+ "Collections.BlobSigningTTL": true,
+ "Collections.BlobTrash": false,
+ "Collections.BlobTrashCheckInterval": false,
+ "Collections.BlobTrashConcurrency": false,
+ "Collections.BlobTrashLifetime": false,
+ "Collections.CollectionVersioning": true,
+ "Collections.DefaultReplication": true,
+ "Collections.DefaultTrashLifetime": true,
+ "Collections.ForwardSlashNameSubstitution": true,
+ "Collections.KeepproxyPermission": false,
+ "Collections.ManagedProperties": true,
+ "Collections.ManagedProperties.*": true,
+ "Collections.ManagedProperties.*.*": true,
+ "Collections.PreserveVersionIfIdle": true,
+ "Collections.S3FolderObjects": true,
+ "Collections.TrashSweepInterval": false,
+ "Collections.TrustAllContent": true,
+ "Collections.WebDAVCache": false,
+ "Collections.WebDAVLogEvents": false,
+ "Collections.WebDAVOutputBuffer": false,
+ "Collections.WebDAVPermission": false,
+ "Containers": true,
+ "Containers.AlwaysUsePreemptibleInstances": true,
+ "Containers.CloudVMs": false,
+ "Containers.CrunchRunArgumentsList": false,
+ "Containers.CrunchRunCommand": false,
+ "Containers.DefaultKeepCacheRAM": true,
+ "Containers.DispatchPrivateKey": false,
+ "Containers.JobsAPI": true,
+ "Containers.JobsAPI.Enable": true,
+ "Containers.LocalKeepBlobBuffersPerVCPU": false,
+ "Containers.LocalKeepLogsToContainerLog": false,
+ "Containers.Logging": false,
+ "Containers.LogReuseDecisions": false,
+ "Containers.LSF": false,
+ "Containers.MaxDispatchAttempts": false,
+ "Containers.MaximumPriceFactor": true,
+ "Containers.MaxRetryAttempts": true,
+ "Containers.MinRetryPeriod": true,
+ "Containers.PreemptiblePriceFactor": false,
+ "Containers.ReserveExtraRAM": true,
+ "Containers.RuntimeEngine": true,
+ "Containers.ShellAccess": true,
+ "Containers.ShellAccess.Admin": true,
+ "Containers.ShellAccess.User": true,
+ "Containers.SLURM": false,
+ "Containers.StaleLockTimeout": false,
+ "Containers.SupportedDockerImageFormats": true,
+ "Containers.SupportedDockerImageFormats.*": true,
+ "InstanceTypes": true,
+ "InstanceTypes.*": true,
+ "InstanceTypes.*.*": true,
+ "InstanceTypes.*.*.*": true,
+ "Login": true,
+ "Login.Google": true,
+ "Login.Google.AlternateEmailAddresses": false,
"Login.Google.AuthenticationRequestParameters": false,
"Login.Google.ClientID": false,
"Login.Google.ClientSecret": false,
"Users.AutoAdminFirstUser": false,
"Users.AutoAdminUserWithEmail": false,
"Users.AutoSetupNewUsers": false,
- "Users.AutoSetupNewUsersWithRepository": false,
"Users.AutoSetupNewUsersWithVmUUID": false,
"Users.AutoSetupUsernameBlacklist": false,
"Users.CanCreateRoleGroups": true,
CrunchDispatchSlurmPath string
WebsocketPath string
KeepproxyPath string
- GitHttpdPath string
KeepBalancePath string
configdata []byte
flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
- flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arvados-git-httpd configuration `file`")
flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
}
if legacyConfigArg != "-legacy-keepproxy-config" {
ldr.KeepproxyPath = ""
}
- if legacyConfigArg != "-legacy-git-httpd-config" {
- ldr.GitHttpdPath = ""
- }
if legacyConfigArg != "-legacy-keepbalance-config" {
ldr.KeepBalancePath = ""
}
ldr.loadOldCrunchDispatchSlurmConfig,
ldr.loadOldWebsocketConfig,
ldr.loadOldKeepproxyConfig,
- ldr.loadOldGitHttpdConfig,
ldr.loadOldKeepBalanceConfig,
)
}
// Test for bug #18076
func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) {
rootctx1, _, _ := s.super.RootClients("z1111")
- _, rootclnt3, _ := s.super.RootClients("z3333")
conn1 := s.super.Conn("z1111")
conn3 := s.super.Conn("z3333")
check.Commentf("incorrect LoginCluster config on cluster %q", cls))
}
- for testCaseNr, testCase := range []struct {
- name string
- withRepository bool
- }{
- {"User without local repository", false},
- {"User with local repository", true},
- } {
- c.Log(c.TestName() + " " + testCase.name)
- // Create some users, request them on the federated cluster so they're cached.
- var users []arvados.User
- for userNr := 0; userNr < 2; userNr++ {
- _, _, _, user := s.super.UserClients("z1111",
- rootctx1,
- c,
- conn1,
- fmt.Sprintf("user%d%d@example.com", testCaseNr, userNr),
- true)
- c.Assert(user.Username, check.Not(check.Equals), "")
- users = append(users, user)
-
- lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
- c.Assert(err, check.Equals, nil)
- userFound := false
- for _, fedUser := range lst.Items {
- if fedUser.UUID == user.UUID {
- c.Assert(fedUser.Username, check.Equals, user.Username)
- userFound = true
- break
- }
- }
- c.Assert(userFound, check.Equals, true)
-
- if testCase.withRepository {
- var repo interface{}
- err = rootclnt3.RequestAndDecode(
- &repo, "POST", "arvados/v1/repositories", nil,
- map[string]interface{}{
- "repository": map[string]string{
- "name": fmt.Sprintf("%s/test", user.Username),
- "owner_uuid": user.UUID,
- },
- },
- )
- c.Assert(err, check.IsNil)
- }
- }
+ // Create some users, request them on the federated cluster so they're cached.
+ var users []arvados.User
+ for userNr := 0; userNr < 2; userNr++ {
+ _, _, _, user := s.super.UserClients("z1111",
+ rootctx1,
+ c,
+ conn1,
+ fmt.Sprintf("user0%d@example.com", userNr),
+ true)
+ c.Assert(user.Username, check.Not(check.Equals), "")
+ users = append(users, user)
- // Swap the usernames
- _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
- UUID: users[0].UUID,
- Attrs: map[string]interface{}{
- "username": "",
- },
- })
- c.Assert(err, check.Equals, nil)
- _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
- UUID: users[1].UUID,
- Attrs: map[string]interface{}{
- "username": users[0].Username,
- },
- })
- c.Assert(err, check.Equals, nil)
- _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
- UUID: users[0].UUID,
- Attrs: map[string]interface{}{
- "username": users[1].Username,
- },
- })
- c.Assert(err, check.Equals, nil)
-
- // Re-request the list on the federated cluster & check for updates
lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
c.Assert(err, check.Equals, nil)
- var user0Found, user1Found bool
- for _, user := range lst.Items {
- if user.UUID == users[0].UUID {
- user0Found = true
- c.Assert(user.Username, check.Equals, users[1].Username)
- } else if user.UUID == users[1].UUID {
- user1Found = true
- c.Assert(user.Username, check.Equals, users[0].Username)
+ userFound := false
+ for _, fedUser := range lst.Items {
+ if fedUser.UUID == user.UUID {
+ c.Assert(fedUser.Username, check.Equals, user.Username)
+ userFound = true
+ break
}
}
- c.Assert(user0Found, check.Equals, true)
- c.Assert(user1Found, check.Equals, true)
+ c.Assert(userFound, check.Equals, true)
+ }
+
+ // Swap the usernames
+ _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
+ UUID: users[0].UUID,
+ Attrs: map[string]interface{}{
+ "username": "",
+ },
+ })
+ c.Assert(err, check.Equals, nil)
+ _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
+ UUID: users[1].UUID,
+ Attrs: map[string]interface{}{
+ "username": users[0].Username,
+ },
+ })
+ c.Assert(err, check.Equals, nil)
+ _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
+ UUID: users[0].UUID,
+ Attrs: map[string]interface{}{
+ "username": users[1].Username,
+ },
+ })
+ c.Assert(err, check.Equals, nil)
+
+ // Re-request the list on the federated cluster & check for updates
+ lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
+ c.Assert(err, check.Equals, nil)
+ var user0Found, user1Found bool
+ for _, user := range lst.Items {
+ if user.UUID == users[0].UUID {
+ user0Found = true
+ c.Assert(user.Username, check.Equals, users[1].Username)
+ } else if user.UUID == users[1].UUID {
+ user1Found = true
+ c.Assert(user.Username, check.Equals, users[0].Username)
+ }
}
+ c.Assert(user0Found, check.Equals, true)
+ c.Assert(user1Found, check.Equals, true)
}
// Test for bug #16263
// OutputPath is a staging directory.
bindmounts[bind] = bindmount{HostPath: tmpfn, ReadOnly: true}
}
-
- case mnt.Kind == "git_tree":
- tmpdir, err := runner.MkTempDir(runner.parentTemp, "git_tree")
- if err != nil {
- return nil, fmt.Errorf("creating temp dir: %v", err)
- }
- err = gitMount(mnt).extractTree(runner.containerClient, tmpdir, token)
- if err != nil {
- return nil, err
- }
- bindmounts[bind] = bindmount{HostPath: tmpdir, ReadOnly: true}
}
}
"git.arvados.org/arvados.git/sdk/go/manifest"
. "gopkg.in/check.v1"
- git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
- git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
)
// Gocheck boilerplate
cr.CleanupDirs()
checkEmpty()
}
-
- // git_tree mounts
- {
- i = 0
- cr.ArvMountPoint = ""
- git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
- cr.token = arvadostest.ActiveToken
- cr.Container.Mounts = make(map[string]arvados.Mount)
- cr.Container.Mounts = map[string]arvados.Mount{
- "/tip": {
- Kind: "git_tree",
- UUID: arvadostest.Repository2UUID,
- Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
- Path: "/",
- },
- "/non-tip": {
- Kind: "git_tree",
- UUID: arvadostest.Repository2UUID,
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- Path: "/",
- },
- }
- cr.Container.OutputPath = "/tmp"
-
- bindmounts, err := cr.SetupMounts()
- c.Check(err, IsNil)
-
- for path, mount := range bindmounts {
- c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
- }
-
- data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
- c.Check(err, IsNil)
- c.Check(string(data), Equals, "\000\001\002\003")
- _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
- c.Check(err, FitsTypeOf, &os.PathError{})
- c.Check(os.IsNotExist(err), Equals, true)
-
- data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
- c.Check(err, IsNil)
- c.Check(string(data), Equals, "\000\001\002\003")
- data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
- c.Check(err, IsNil)
- c.Check(string(data), Equals, "testfile\n")
-
- cr.CleanupDirs()
- checkEmpty()
- }
}
func (s *TestSuite) TestStdout(c *C) {
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package crunchrun
-
-import (
- "fmt"
- "net/url"
- "os"
- "path/filepath"
- "regexp"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "gopkg.in/src-d/go-billy.v4/osfs"
- git "gopkg.in/src-d/go-git.v4"
- git_config "gopkg.in/src-d/go-git.v4/config"
- git_plumbing "gopkg.in/src-d/go-git.v4/plumbing"
- git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
- "gopkg.in/src-d/go-git.v4/storage/memory"
-)
-
-type gitMount arvados.Mount
-
-var (
- sha1re = regexp.MustCompile(`^[0-9a-f]{40}$`)
- repoUUIDre = regexp.MustCompile(`^[0-9a-z]{5}-s0uqq-[0-9a-z]{15}$`)
-)
-
-func (gm gitMount) validate() error {
- if gm.Path != "" && gm.Path != "/" {
- return fmt.Errorf("cannot mount git_tree with path %q -- only \"/\" is supported", gm.Path)
- }
- if !sha1re.MatchString(gm.Commit) {
- return fmt.Errorf("cannot mount git_tree with commit %q -- must be a 40-char SHA1", gm.Commit)
- }
- if gm.RepositoryName != "" || gm.GitURL != "" {
- return fmt.Errorf("cannot mount git_tree -- repository_name and git_url must be empty")
- }
- if !repoUUIDre.MatchString(gm.UUID) {
- return fmt.Errorf("cannot mount git_tree with uuid %q -- must be a repository UUID", gm.UUID)
- }
- if gm.Writable {
- return fmt.Errorf("writable git_tree mount is not supported")
- }
- return nil
-}
-
-// ExtractTree extracts the specified tree into dir, which is an
-// existing empty local directory.
-func (gm gitMount) extractTree(ac *arvados.Client, dir string, token string) error {
- err := gm.validate()
- if err != nil {
- return err
- }
- dd, err := ac.DiscoveryDocument()
- if err != nil {
- return fmt.Errorf("error getting discovery document: %w", err)
- }
- u, err := url.Parse(dd.GitURL)
- if err != nil {
- return fmt.Errorf("parse gitUrl %q: %s", dd.GitURL, err)
- }
- u, err = u.Parse("/" + gm.UUID + ".git")
- if err != nil {
- return fmt.Errorf("build git url from %q, %q: %s", dd.GitURL, gm.UUID, err)
- }
- store := memory.NewStorage()
- repo, err := git.Init(store, osfs.New(dir))
- if err != nil {
- return fmt.Errorf("init repo: %s", err)
- }
- _, err = repo.CreateRemote(&git_config.RemoteConfig{
- Name: "origin",
- URLs: []string{u.String()},
- })
- if err != nil {
- return fmt.Errorf("create remote %q: %s", u.String(), err)
- }
- err = repo.Fetch(&git.FetchOptions{
- RemoteName: "origin",
- Auth: &git_http.BasicAuth{
- Username: "none",
- Password: token,
- },
- })
- if err != nil {
- return fmt.Errorf("git fetch %q: %s", u.String(), err)
- }
- wt, err := repo.Worktree()
- if err != nil {
- return fmt.Errorf("worktree failed: %s", err)
- }
- err = wt.Checkout(&git.CheckoutOptions{
- Hash: git_plumbing.NewHash(gm.Commit),
- })
- if err != nil {
- return fmt.Errorf("checkout failed: %s", err)
- }
- err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- // copy user rx bits to group and other, in case
- // prevailing umask is more restrictive than 022
- mode := info.Mode()
- mode = mode | ((mode >> 3) & 050) | ((mode >> 6) & 5)
- return os.Chmod(path, mode)
- })
- if err != nil {
- return fmt.Errorf("chmod -R %q: %s", dir, err)
- }
- return nil
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package crunchrun
-
-import (
- "io/ioutil"
- "os"
- "path/filepath"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/arvadostest"
- check "gopkg.in/check.v1"
- git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
- git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
-)
-
-type GitMountSuite struct {
- tmpdir string
-}
-
-var _ = check.Suite(&GitMountSuite{})
-
-func (s *GitMountSuite) SetUpTest(c *check.C) {
- var err error
- s.tmpdir, err = ioutil.TempDir("", "")
- c.Assert(err, check.IsNil)
- git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
-}
-
-func (s *GitMountSuite) TearDownTest(c *check.C) {
- err := os.RemoveAll(s.tmpdir)
- c.Check(err, check.IsNil)
-}
-
-// Commit fd3531f is crunch-run-tree-test
-func (s *GitMountSuite) TestExtractTree(c *check.C) {
- gm := gitMount{
- Path: "/",
- UUID: arvadostest.Repository2UUID,
- Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
- }
- ac := arvados.NewClientFromEnv()
- err := gm.extractTree(ac, s.tmpdir, arvadostest.ActiveToken)
- c.Check(err, check.IsNil)
-
- fnm := filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0644")
- data, err := ioutil.ReadFile(fnm)
- c.Check(err, check.IsNil)
- c.Check(data, check.DeepEquals, []byte{0, 1, 2, 3})
- fi, err := os.Stat(fnm)
- c.Check(err, check.IsNil)
- if err == nil {
- c.Check(fi.Mode(), check.Equals, os.FileMode(0644))
- }
-
- fnm = filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0755")
- data, err = ioutil.ReadFile(fnm)
- c.Check(err, check.IsNil)
- c.Check(string(data), check.DeepEquals, "#!/bin/sh\nexec echo OK\n")
- fi, err = os.Stat(fnm)
- c.Check(err, check.IsNil)
- if err == nil {
- c.Check(fi.Mode(), check.Equals, os.FileMode(0755))
- }
-
- // Ensure there's no extra stuff like a ".git" dir
- s.checkTmpdirContents(c, []string{"dir1"})
-
- // Ensure tmpdir is world-readable and world-executable so the
- // UID inside the container can use it.
- fi, err = os.Stat(s.tmpdir)
- c.Check(err, check.IsNil)
- c.Check(fi.Mode()&os.ModePerm, check.Equals, os.FileMode(0755))
-}
-
-// Commit 5ebfab0 is not the tip of any branch or tag, but is
-// reachable in branch "crunch-run-non-tip-test".
-func (s *GitMountSuite) TestExtractNonTipCommit(c *check.C) {
- gm := gitMount{
- UUID: arvadostest.Repository2UUID,
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- }
- err := gm.extractTree(arvados.NewClientFromEnv(), s.tmpdir, arvadostest.ActiveToken)
- c.Check(err, check.IsNil)
-
- fnm := filepath.Join(s.tmpdir, "file only on testbranch")
- data, err := ioutil.ReadFile(fnm)
- c.Check(err, check.IsNil)
- c.Check(string(data), check.DeepEquals, "testfile\n")
-}
-
-func (s *GitMountSuite) TestNonexistentRepository(c *check.C) {
- gm := gitMount{
- Path: "/",
- UUID: "zzzzz-s0uqq-nonexistentrepo",
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- }
- err := gm.extractTree(arvados.NewClientFromEnv(), s.tmpdir, arvadostest.ActiveToken)
- c.Check(err, check.NotNil)
- c.Check(err, check.ErrorMatches, ".*repository not found.*")
-
- s.checkTmpdirContents(c, []string{})
-}
-
-func (s *GitMountSuite) TestNonexistentCommit(c *check.C) {
- gm := gitMount{
- Path: "/",
- UUID: arvadostest.Repository2UUID,
- Commit: "bb66b6bb6b6bbb6b6b6b66b6b6b6b6b6b6b6b66b",
- }
- err := gm.extractTree(arvados.NewClientFromEnv(), s.tmpdir, arvadostest.ActiveToken)
- c.Check(err, check.NotNil)
- c.Check(err, check.ErrorMatches, ".*object not found.*")
-
- s.checkTmpdirContents(c, []string{})
-}
-
-func (s *GitMountSuite) TestGitUrlDiscoveryFails(c *check.C) {
- delete(discoveryMap, "gitUrl")
- gm := gitMount{
- Path: "/",
- UUID: arvadostest.Repository2UUID,
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- }
- err := gm.extractTree(&arvados.Client{}, s.tmpdir, arvadostest.ActiveToken)
- c.Check(err, check.ErrorMatches, ".*error getting discovery doc.*")
-}
-
-func (s *GitMountSuite) TestInvalid(c *check.C) {
- for _, trial := range []struct {
- gm gitMount
- matcher string
- }{
- {
- gm: gitMount{
- Path: "/",
- UUID: arvadostest.Repository2UUID,
- Commit: "abc123",
- },
- matcher: ".*SHA1.*",
- },
- {
- gm: gitMount{
- Path: "/",
- UUID: arvadostest.Repository2UUID,
- RepositoryName: arvadostest.Repository2Name,
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- },
- matcher: ".*repository_name.*",
- },
- {
- gm: gitMount{
- Path: "/",
- GitURL: "https://localhost:0/" + arvadostest.Repository2Name + ".git",
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- },
- matcher: ".*git_url.*",
- },
- {
- gm: gitMount{
- Path: "/dir1/",
- UUID: arvadostest.Repository2UUID,
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- },
- matcher: ".*path.*",
- },
- {
- gm: gitMount{
- Path: "/",
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- },
- matcher: ".*UUID.*",
- },
- {
- gm: gitMount{
- Path: "/",
- UUID: arvadostest.Repository2UUID,
- Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
- Writable: true,
- },
- matcher: ".*writable.*",
- },
- } {
- err := trial.gm.extractTree(arvados.NewClientFromEnv(), s.tmpdir, arvadostest.ActiveToken)
- c.Check(err, check.NotNil)
- s.checkTmpdirContents(c, []string{})
-
- err = trial.gm.validate()
- c.Check(err, check.ErrorMatches, trial.matcher)
- }
-}
-
-func (s *GitMountSuite) checkTmpdirContents(c *check.C, expect []string) {
- f, err := os.Open(s.tmpdir)
- c.Check(err, check.IsNil)
- names, err := f.Readdirnames(-1)
- c.Check(err, check.IsNil)
- c.Check(names, check.DeepEquals, expect)
-}
"curl",
"fuse",
"git",
- "gitolite3",
"graphviz",
"haveged",
"libcurl3-gnutls",
Keepbalance:
InternalURLs:
"http://0.0.0.0:9019/": {}
- GitHTTP:
- InternalURLs:
- "http://0.0.0.0:9005/": {}
- ExternalURL: {{printf "%q" ( print "https://" .Domain ":4445/" ) }}
DispatchCloud:
InternalURLs:
"http://0.0.0.0:9006/": {}
loader := config.NewLoader(stdin, log)
loader.SetupFlags(flags)
- // prog is [keepstore, keep-web, git-httpd, ...] but the
+ // prog is [keepstore, keep-web, ...] but the
// legacy config flags are [-legacy-keepstore-config,
- // -legacy-keepweb-config, -legacy-git-httpd-config, ...]
+ // -legacy-keepweb-config, ...]
legacyFlag := "-legacy-" + strings.Replace(prog, "keep-", "keep", 1) + "-config"
args = loader.MungeLegacyConfigArgs(log, args, legacyFlag)
WebDAVLogEvents bool
WebDAVOutputBuffer ByteSize
}
- Git struct {
- GitCommand string
- GitoliteHome string
- Repositories string
- }
Login struct {
LDAP struct {
Enable bool
AutoAdminFirstUser bool
AutoAdminUserWithEmail string
AutoSetupNewUsers bool
- AutoSetupNewUsersWithRepository bool
AutoSetupNewUsersWithVmUUID string
AutoSetupUsernameBlacklist StringSet
EmailSubjectPrefix string
DispatchCloud Service
DispatchLSF Service
DispatchSLURM Service
- GitHTTP Service
- GitSSH Service
Health Service
Keepbalance Service
Keepproxy Service
LocalKeepLogsToContainerLog string
JobsAPI struct {
- Enable string
- GitInternalDir string
+ Enable string
}
Logging struct {
MaxAge Duration
ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
- ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
ServiceNameHealth ServiceName = "arvados-health"
ServiceNameKeepbalance ServiceName = "keep-balance"
ServiceNameKeepproxy ServiceName = "keepproxy"
ServiceNameDispatchCloud: svcs.DispatchCloud,
ServiceNameDispatchLSF: svcs.DispatchLSF,
ServiceNameDispatchSLURM: svcs.DispatchSLURM,
- ServiceNameGitHTTP: svcs.GitHTTP,
ServiceNameHealth: svcs.Health,
ServiceNameKeepbalance: svcs.Keepbalance,
ServiceNameKeepproxy: svcs.Keepproxy,
Content interface{} `json:"content"`
ExcludeFromOutput bool `json:"exclude_from_output"`
Capacity int64 `json:"capacity"`
- Commit string `json:"commit"` // only if kind=="git_tree"
- RepositoryName string `json:"repository_name"` // only if kind=="git_tree"
- GitURL string `json:"git_url"` // only if kind=="git_tree"
}
type CUDARuntimeConstraints struct {
// A ClientPool is a pool of ArvadosClients. This is useful for
// applications that make API calls using a dynamic set of tokens,
// like web services that pass through their own clients'
-// credentials. See arvados-git-httpd for an example, and sync.Pool
-// for more information about garbage collection.
+// credentials. See sync.Pool for more information about garbage
+// collection.
type ClientPool struct {
// Initialize new clients by copying this one.
Prototype *ArvadosClient
&svcs.DispatchCloud,
&svcs.DispatchLSF,
&svcs.DispatchSLURM,
- &svcs.GitHTTP,
&svcs.Keepbalance,
&svcs.Keepproxy,
&svcs.Keepstore,
c['manifest_text'] = dst_manifest.getvalue()
return create_collection_from(c, src, dst, args)
-def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_http_opt):
- r = api.repositories().list(
- filters=[['name', '=', repo_name]]).execute(num_retries=retries)
- if r['items_available'] != 1:
- raise Exception('cannot identify repo {}; {} repos found'
- .format(repo_name, r['items_available']))
-
- https_url = [c for c in r['items'][0]["clone_urls"] if c.startswith("https:")]
- http_url = [c for c in r['items'][0]["clone_urls"] if c.startswith("http:")]
- other_url = [c for c in r['items'][0]["clone_urls"] if not c.startswith("http")]
-
- priority = https_url + other_url + http_url
-
- for url in priority:
- if url.startswith("http"):
- u = urllib.parse.urlsplit(url)
- baseurl = urllib.parse.urlunsplit((u.scheme, u.netloc, "", "", ""))
- git_config = ["-c", "credential.%s/.username=none" % baseurl,
- "-c", "credential.%s/.helper=!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred" % baseurl]
- else:
- git_config = []
-
- try:
- logger.debug("trying %s", url)
- subprocess.run(
- ['git', *git_config, 'ls-remote', url],
- check=True,
- env={
- 'ARVADOS_API_TOKEN': api.api_token,
- 'GIT_ASKPASS': '/bin/false',
- 'HOME': os.environ['HOME'],
- },
- stdout=subprocess.DEVNULL,
- )
- except subprocess.CalledProcessError:
- pass
- else:
- git_url = url
- break
- else:
- raise Exception('Cannot access git repository, tried {}'
- .format(priority))
-
- if git_url.startswith("http:"):
- if allow_insecure_http:
- logger.warning("Using insecure git url %s but will allow this because %s", git_url, allow_insecure_http_opt)
- else:
- raise Exception("Refusing to use insecure git url %s, use %s if you really want this." % (git_url, allow_insecure_http_opt))
-
- return (git_url, git_config)
-
-
def copy_docker_image(docker_image, docker_image_tag, src, dst, args):
"""Copy the docker image identified by docker_image and
docker_image_tag from src to dst. Create appropriate
proxy_http_version 1.1;
}
}
- upstream arv-git-http {
- server {{UPSTREAMHOST}}:{{GITPORT}};
- }
- server {
- listen {{LISTENHOST}}:{{GITSSLPORT}} ssl;
- server_name arv-git-http git.*;
- ssl_certificate "{{SSLCERT}}";
- ssl_certificate_key "{{SSLKEY}}";
- location / {
- proxy_pass http://arv-git-http;
- proxy_set_header Host $http_host;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto https;
- proxy_redirect off;
- }
- }
upstream keepproxy {
server {{UPSTREAMHOST}}:{{KEEPPROXYPORT}};
}
return
kill_server_pid(_pidfile('keepproxy'))
-def run_arv_git_httpd():
- if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
- return
- stop_arv_git_httpd()
-
- gitport = internal_port_from_config("GitHTTP")
- env = os.environ.copy()
- env.pop('ARVADOS_API_TOKEN', None)
- logf = open(_logfilename('githttpd'), WRITE_MODE)
- agh = subprocess.Popen(['arvados-server', 'git-httpd'],
- env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
- _detachedSubprocesses.append(agh)
- with open(_pidfile('githttpd'), 'w') as f:
- f.write(str(agh.pid))
- _wait_until_port_listens(gitport)
-
-def stop_arv_git_httpd():
- if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
- return
- kill_server_pid(_pidfile('githttpd'))
-
def run_keep_web():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
return
nginxconf['KEEPWEBSSLPORT'] = external_port_from_config("WebDAV")
nginxconf['KEEPPROXYPORT'] = internal_port_from_config("Keepproxy")
nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
- nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
- nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
nginxconf['HEALTHPORT'] = internal_port_from_config("Health")
nginxconf['HEALTHSSLPORT'] = external_port_from_config("Health")
nginxconf['WSPORT'] = internal_port_from_config("Websocket")
workbench1_external_port = find_available_port()
workbench2_port = find_available_port()
workbench2_external_port = find_available_port()
- git_httpd_port = find_available_port()
- git_httpd_external_port = find_available_port()
health_httpd_port = find_available_port()
health_httpd_external_port = find_available_port()
keepproxy_port = find_available_port()
"http://%s:%s"%(localhost, workbench2_port): {},
},
},
- "GitHTTP": {
- "ExternalURL": "https://%s:%s" % (localhost, git_httpd_external_port),
- "InternalURLs": {
- "http://%s:%s"%(localhost, git_httpd_port): {}
- },
- },
"Health": {
"ExternalURL": "https://%s:%s" % (localhost, health_httpd_external_port),
"InternalURLs": {
"ForwardSlashNameSubstitution": "/",
"TrashSweepInterval": "-1s",
},
- "Git": {
- "Repositories": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git', 'test'),
- },
"Containers": {
- "JobsAPI": {
- "GitInternalDir": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'internal.git'),
- },
"LocalKeepBlobBuffersPerVCPU": 0,
"Logging": {
"SweepInterval": 0, # disable, otherwise test cases can't acquire dblock
'start_keep', 'stop_keep',
'start_keep_proxy', 'stop_keep_proxy',
'start_keep-web', 'stop_keep-web',
- 'start_githttpd', 'stop_githttpd',
'start_nginx', 'stop_nginx', 'setup_config',
]
parser = argparse.ArgumentParser()
run_keep_proxy()
elif args.action == 'stop_keep_proxy':
stop_keep_proxy()
- elif args.action == 'start_githttpd':
- run_arv_git_httpd()
- elif args.action == 'stop_githttpd':
- stop_arv_git_httpd()
elif args.action == 'start_keep-web':
run_keep_web()
elif args.action == 'stop_keep-web':
workbenchUrl: Rails.configuration.Services.Workbench1.ExternalURL.to_s,
workbench2Url: Rails.configuration.Services.Workbench2.ExternalURL.to_s,
keepWebServiceUrl: Rails.configuration.Services.WebDAV.ExternalURL.to_s,
- gitUrl: Rails.configuration.Services.GitHTTP.ExternalURL.to_s,
parameters: {
alt: {
type: "string",
arvcfg.declare_config "ClusterID", NonemptyString, :uuid_prefix
arvcfg.declare_config "ManagementToken", String, :ManagementToken
arvcfg.declare_config "SystemRootToken", String
-arvcfg.declare_config "Git.Repositories", String, :git_repositories_dir
arvcfg.declare_config "API.DisabledAPIs", Hash, :disable_api_methods, ->(cfg, k, v) { arrayToHash cfg, "API.DisabledAPIs", v }
arvcfg.declare_config "API.MaxRequestSize", Integer, :max_request_size
arvcfg.declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
arvcfg.declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
arvcfg.declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
arvcfg.declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
-arvcfg.declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
arvcfg.declare_config "Users.AutoSetupUsernameBlacklist", Hash, :auto_setup_name_blacklist, ->(cfg, k, v) { arrayToHash cfg, "Users.AutoSetupUsernameBlacklist", v }
arvcfg.declare_config "Users.NewUsersAreActive", Boolean, :new_users_are_active
arvcfg.declare_config "Users.AutoAdminUserWithEmail", String, :auto_admin_user
arvcfg.declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address
arvcfg.declare_config "Services.Websocket.ExternalURL", URI, :websocket_address
arvcfg.declare_config "Services.WebDAV.ExternalURL", URI, :keep_web_service_url
-arvcfg.declare_config "Services.GitHTTP.ExternalURL", URI, :git_repo_https_base
-arvcfg.declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base, ->(cfg, k, v) { ConfigLoader.set_cfg cfg, "Services.GitSSH.ExternalURL", "ssh://#{v}" }
arvcfg.declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
h = if cfg["RemoteClusters"] then
cfg["RemoteClusters"].deep_dup
+++ /dev/null
-#!/usr/bin/env ruby
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'rubygems'
-require 'pp'
-require 'arvados'
-require 'tempfile'
-require 'yaml'
-require 'fileutils'
-
-# This script does the actual gitolite config management on disk.
-#
-# Ward Vandewege <ward@curii.com>
-
-# Default is development
-production = ARGV[0] == "production"
-
-ENV["RAILS_ENV"] = "development"
-ENV["RAILS_ENV"] = "production" if production
-
-DEBUG = 1
-
-# load and merge in the environment-specific application config info
-# if present, overriding base config parameters as specified
-path = File.absolute_path('../../config/arvados-clients.yml', __FILE__)
-if File.exist?(path) then
- cp_config = File.open(path) do |f|
- YAML.safe_load(f, filename: path)[ENV['RAILS_ENV']]
- end
-else
- puts "Please create a\n #{path}\n file"
- exit 1
-end
-
-gitolite_url = cp_config['gitolite_url']
-gitolite_arvados_git_user_key = cp_config['gitolite_arvados_git_user_key']
-
-gitolite_tmpdir = cp_config['gitolite_tmp']
-gitolite_admin = File.join(gitolite_tmpdir, 'gitolite-admin')
-gitolite_admin_keydir = File.join(gitolite_admin, 'keydir')
-gitolite_keydir = File.join(gitolite_admin, 'keydir', 'arvados')
-
-ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host']
-ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token']
-if cp_config['arvados_api_host_insecure']
- ENV['ARVADOS_API_HOST_INSECURE'] = 'true'
-else
- ENV.delete('ARVADOS_API_HOST_INSECURE')
-end
-
-def ensure_directory(path, mode)
- begin
- Dir.mkdir(path, mode)
- rescue Errno::EEXIST
- end
-end
-
-def replace_file(path, contents)
- unlink_now = true
- dirname, basename = File.split(path)
- FileUtils.mkpath(dirname)
- new_file = Tempfile.new([basename, ".tmp"], dirname)
- begin
- new_file.write(contents)
- new_file.flush
- File.rename(new_file, path)
- unlink_now = false
- ensure
- new_file.close(unlink_now)
- end
-end
-
-def file_has_contents?(path, contents)
- begin
- IO.read(path) == contents
- rescue Errno::ENOENT
- false
- end
-end
-
-module TrackCommitState
- module ClassMethods
- # Note that all classes that include TrackCommitState will have
- # @@need_commit = true if any of them set it. Since this flag reports
- # a boolean state of the underlying git repository, that's OK in the
- # current implementation.
- @@need_commit = false
-
- def changed?
- @@need_commit
- end
-
- def ensure_in_git(path, contents)
- unless file_has_contents?(path, contents)
- replace_file(path, contents)
- system("git", "add", path)
- @@need_commit = true
- end
- end
- end
-
- def ensure_in_git(path, contents)
- self.class.ensure_in_git(path, contents)
- end
-
- def self.included(base)
- base.extend(ClassMethods)
- end
-end
-
-class UserSSHKeys
- include TrackCommitState
-
- def initialize(user_keys_map, key_dir)
- @user_keys_map = user_keys_map
- @key_dir = key_dir
- @installed = {}
- end
-
- def install(filename, pubkey)
- unless pubkey.nil?
- key_path = File.join(@key_dir, filename)
- ensure_in_git(key_path, pubkey)
- end
- @installed[filename] = true
- end
-
- def ensure_keys_for_user(user_uuid)
- return unless key_list = @user_keys_map.delete(user_uuid)
- key_list.map { |k| k[:public_key] }.compact.each_with_index do |pubkey, ii|
- # Handle putty-style ssh public keys
- pubkey.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
- pubkey.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
- pubkey.gsub!(/\n/,'')
- pubkey.strip!
- install("#{user_uuid}@#{ii}.pub", pubkey)
- end
- end
-
- def installed?(filename)
- @installed[filename]
- end
-end
-
-class Repository
- include TrackCommitState
-
- @@aliases = {}
-
- def initialize(arv_repo, user_keys)
- @arv_repo = arv_repo
- @user_keys = user_keys
- end
-
- def self.ensure_system_config(conf_root)
- ensure_in_git(File.join(conf_root, "conf", "gitolite.conf"),
- %Q{include "auto/*.conf"\ninclude "admin/*.conf"\n})
- ensure_in_git(File.join(conf_root, "arvadosaliases.pl"), alias_config)
-
- conf_path = File.join(conf_root, "conf", "admin", "arvados.conf")
- conf_file = %Q{
-@arvados_git_user = arvados_git_user
-
-repo gitolite-admin
- RW = @arvados_git_user
-
-}
- ensure_directory(File.dirname(conf_path), 0755)
- ensure_in_git(conf_path, conf_file)
- end
-
- def ensure_config(conf_root)
- if name and (File.exist?(auto_conf_path(conf_root, name)))
- # This gitolite installation knows the repository by name, rather than
- # UUID. Leave it configured that way until a separate migration is run.
- basename = name
- else
- basename = uuid
- @@aliases[name] = uuid unless name.nil?
- end
- conf_file = "\nrepo #{basename}\n"
- @arv_repo[:user_permissions].sort.each do |user_uuid, perm|
- conf_file += "\t#{perm[:gitolite_permissions]}\t= #{user_uuid}\n"
- @user_keys.ensure_keys_for_user(user_uuid)
- end
- ensure_in_git(auto_conf_path(conf_root, basename), conf_file)
- end
-
- private
-
- def auto_conf_path(conf_root, basename)
- File.join(conf_root, "conf", "auto", "#{basename}.conf")
- end
-
- def uuid
- @arv_repo[:uuid]
- end
-
- def name
- if @arv_repo[:name].nil?
- nil
- else
- @clean_name ||=
- @arv_repo[:name].sub(/^[^A-Za-z]+/, "").gsub(/[^\w\.\/]/, "")
- end
- end
-
- def self.alias_config
- conf_s = "{\n"
- @@aliases.sort.each do |(repo_name, repo_uuid)|
- conf_s += "\t'#{repo_name}' \t=> '#{repo_uuid}',\n"
- end
- conf_s += "};\n"
- conf_s
- end
-end
-
-begin
- # Get our local gitolite-admin repo up to snuff
- if not File.exist?(gitolite_admin) then
- ensure_directory(gitolite_tmpdir, 0700)
- Dir.chdir(gitolite_tmpdir)
- `git clone #{gitolite_url}`
- Dir.chdir(gitolite_admin)
- else
- Dir.chdir(gitolite_admin)
- `git pull`
- end
-
- arv = Arvados.new
- permissions = arv.repository.get_all_permissions
-
- ensure_directory(gitolite_keydir, 0700)
- admin_user_ssh_keys = UserSSHKeys.new(permissions[:user_keys], gitolite_admin_keydir)
- # Make sure the arvados_git_user key is installed; put it in gitolite_admin_keydir
- # because that is where gitolite will try to put it if we do not.
- admin_user_ssh_keys.install('arvados_git_user.pub', gitolite_arvados_git_user_key)
-
- user_ssh_keys = UserSSHKeys.new(permissions[:user_keys], gitolite_keydir)
- permissions[:repositories].each do |repo_record|
- repo = Repository.new(repo_record, user_ssh_keys)
- repo.ensure_config(gitolite_admin)
- end
- Repository.ensure_system_config(gitolite_admin)
-
- # Clean up public key files that should not be present
- Dir.chdir(gitolite_keydir)
- stale_keys = Dir.glob('*.pub').reject do |key_file|
- user_ssh_keys.installed?(key_file)
- end
- if stale_keys.any?
- stale_keys.each { |key_file| puts "Extra file #{key_file}" }
- system("git", "rm", "--quiet", *stale_keys)
- end
-
- if UserSSHKeys.changed? or Repository.changed? or stale_keys.any?
- message = "#{Time.now().to_s}: update from API"
- Dir.chdir(gitolite_admin)
- `git add --all`
- `git commit -m '#{message}'`
- `git push`
- end
-
-rescue => bang
- puts "Error: " + bang.to_s
- puts bang.backtrace.join("\n")
- exit 1
-end
-
+++ /dev/null
-#!/usr/bin/env ruby
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-#
-# Prior to April 2015, Arvados Gitolite integration stored repositories by
-# name. To improve user repository management, we switched to storing
-# repositories by UUID, and aliasing them to names. This makes it easy to
-# have rich name hierarchies, and allow users to rename repositories.
-#
-# This script will migrate a name-based Gitolite configuration to a UUID-based
-# one. To use it:
-#
-# 1. Change the value of REPOS_DIR below, if needed.
-# 2. Install this script in the same directory as `update-gitolite.rb`.
-# 3. Ensure that no *other* users can access Gitolite: edit gitolite's
-# authorized_keys file so it only contains the arvados_git_user key,
-# and disable the update-gitolite cron job.
-# 4. Run this script: `ruby migrate-gitolite-to-uuid-storage.rb production`.
-# 5. Undo step 3.
-
-require 'rubygems'
-require 'pp'
-require 'arvados'
-require 'tempfile'
-require 'yaml'
-
-REPOS_DIR = "/var/lib/gitolite/repositories"
-
-# Default is development
-production = ARGV[0] == "production"
-
-ENV["RAILS_ENV"] = "development"
-ENV["RAILS_ENV"] = "production" if production
-
-DEBUG = 1
-
-# load and merge in the environment-specific application config info
-# if present, overriding base config parameters as specified
-path = File.dirname(__FILE__) + '/config/arvados-clients.yml'
-if File.exist?(path) then
- cp_config = File.open(path) do |f|
- YAML.safe_load(f, filename: path)[ENV['RAILS_ENV']]
- end
-else
- puts "Please create a\n " + File.dirname(__FILE__) + "/config/arvados-clients.yml\n file"
- exit 1
-end
-
-gitolite_url = cp_config['gitolite_url']
-gitolite_arvados_git_user_key = cp_config['gitolite_arvados_git_user_key']
-
-gitolite_tmpdir = File.join(File.absolute_path(File.dirname(__FILE__)),
- cp_config['gitolite_tmp'])
-gitolite_admin = File.join(gitolite_tmpdir, 'gitolite-admin')
-gitolite_keydir = File.join(gitolite_admin, 'keydir', 'arvados')
-
-ENV['ARVADOS_API_HOST'] = cp_config['arvados_api_host']
-ENV['ARVADOS_API_TOKEN'] = cp_config['arvados_api_token']
-if cp_config['arvados_api_host_insecure']
- ENV['ARVADOS_API_HOST_INSECURE'] = 'true'
-else
- ENV.delete('ARVADOS_API_HOST_INSECURE')
-end
-
-def ensure_directory(path, mode)
- begin
- Dir.mkdir(path, mode)
- rescue Errno::EEXIST
- end
-end
-
-def replace_file(path, contents)
- unlink_now = true
- dirname, basename = File.split(path)
- new_file = Tempfile.new([basename, ".tmp"], dirname)
- begin
- new_file.write(contents)
- new_file.flush
- File.rename(new_file, path)
- unlink_now = false
- ensure
- new_file.close(unlink_now)
- end
-end
-
-def file_has_contents?(path, contents)
- begin
- IO.read(path) == contents
- rescue Errno::ENOENT
- false
- end
-end
-
-module TrackCommitState
- module ClassMethods
- # Note that all classes that include TrackCommitState will have
- # @@need_commit = true if any of them set it. Since this flag reports
- # a boolean state of the underlying git repository, that's OK in the
- # current implementation.
- @@need_commit = false
-
- def changed?
- @@need_commit
- end
-
- def ensure_in_git(path, contents)
- unless file_has_contents?(path, contents)
- replace_file(path, contents)
- system("git", "add", path)
- @@need_commit = true
- end
- end
- end
-
- def ensure_in_git(path, contents)
- self.class.ensure_in_git(path, contents)
- end
-
- def self.included(base)
- base.extend(ClassMethods)
- end
-end
-
-class Repository
- include TrackCommitState
-
- @@aliases = {}
-
- def initialize(arv_repo)
- @arv_repo = arv_repo
- end
-
- def self.ensure_system_config(conf_root)
- ensure_in_git(File.join(conf_root, "arvadosaliases.pl"), alias_config)
- end
-
- def self.rename_repos(repos_root)
- @@aliases.each_pair do |uuid, name|
- begin
- File.rename(File.join(repos_root, "#{name}.git/"),
- File.join(repos_root, "#{uuid}.git"))
- rescue Errno::ENOENT
- end
- if name == "arvados"
- Dir.chdir(repos_root) { File.symlink("#{uuid}.git/", "arvados.git") }
- end
- end
- end
-
- def ensure_config(conf_root)
- return if name.nil?
- @@aliases[uuid] = name
- name_conf_path = auto_conf_path(conf_root, name)
- return unless File.exist?(name_conf_path)
- conf_file = IO.read(name_conf_path)
- conf_file.gsub!(/^repo #{Regexp.escape(name)}$/m, "repo #{uuid}")
- ensure_in_git(auto_conf_path(conf_root, uuid), conf_file)
- File.unlink(name_conf_path)
- system("git", "rm", "--quiet", name_conf_path)
- end
-
- private
-
- def auto_conf_path(conf_root, basename)
- File.join(conf_root, "conf", "auto", "#{basename}.conf")
- end
-
- def uuid
- @arv_repo[:uuid]
- end
-
- def name
- if @arv_repo[:name].nil?
- nil
- else
- @clean_name ||=
- @arv_repo[:name].sub(/^[^A-Za-z]+/, "").gsub(/[^\w\.\/]/, "")
- end
- end
-
- def self.alias_config
- conf_s = "{\n"
- @@aliases.sort.each do |(repo_name, repo_uuid)|
- conf_s += "\t'#{repo_name}' \t=> '#{repo_uuid}',\n"
- end
- conf_s += "};\n"
- conf_s
- end
-end
-
-begin
- # Get our local gitolite-admin repo up to snuff
- if not File.exist?(gitolite_admin) then
- ensure_directory(gitolite_tmpdir, 0700)
- Dir.chdir(gitolite_tmpdir)
- `git clone #{gitolite_url}`
- Dir.chdir(gitolite_admin)
- else
- Dir.chdir(gitolite_admin)
- `git pull`
- end
-
- arv = Arvados.new
- permissions = arv.repository.get_all_permissions
-
- permissions[:repositories].each do |repo_record|
- repo = Repository.new(repo_record)
- repo.ensure_config(gitolite_admin)
- end
- Repository.ensure_system_config(gitolite_admin)
-
- message = "#{Time.now().to_s}: migrate to storing repositories by UUID"
- Dir.chdir(gitolite_admin)
- `git add --all`
- `git commit -m '#{message}'`
- Repository.rename_repos(REPOS_DIR)
- `git push`
-
-rescue => bang
- puts "Error: " + bang.to_s
- puts bang.backtrace.join("\n")
- exit 1
-end
-
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "errors"
- "log"
- "net/http"
- "os"
- "regexp"
- "strings"
- "time"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/arvadosclient"
- "git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
- "github.com/sirupsen/logrus"
-)
-
-type authHandler struct {
- handler http.Handler
- clientPool *arvadosclient.ClientPool
- cluster *arvados.Cluster
-}
-
-func (h *authHandler) CheckHealth() error {
- return nil
-}
-
-func (h *authHandler) Done() <-chan struct{} {
- return nil
-}
-
-func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
- var statusCode int
- var statusText string
- var apiToken string
-
- w := httpserver.WrapResponseWriter(wOrig)
-
- if r.Method == "OPTIONS" {
- method := r.Header.Get("Access-Control-Request-Method")
- if method != "GET" && method != "POST" {
- w.WriteHeader(http.StatusMethodNotAllowed)
- return
- }
- w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
- w.Header().Set("Access-Control-Allow-Methods", "GET, POST")
- w.Header().Set("Access-Control-Allow-Origin", "*")
- w.Header().Set("Access-Control-Max-Age", "86400")
- w.WriteHeader(http.StatusOK)
- return
- }
-
- if r.Header.Get("Origin") != "" {
- // Allow simple cross-origin requests without user
- // credentials ("user credentials" as defined by CORS,
- // i.e., cookies, HTTP authentication, and client-side
- // SSL certificates. See
- // http://www.w3.org/TR/cors/#user-credentials).
- w.Header().Set("Access-Control-Allow-Origin", "*")
- }
-
- defer func() {
- if w.WroteStatus() == 0 {
- // Nobody has called WriteHeader yet: that
- // must be our job.
- w.WriteHeader(statusCode)
- if statusCode >= 400 {
- w.Write([]byte(statusText))
- }
- }
- }()
-
- creds := auth.CredentialsFromRequest(r)
- if len(creds.Tokens) == 0 {
- statusCode, statusText = http.StatusUnauthorized, "no credentials provided"
- w.Header().Add("WWW-Authenticate", "Basic realm=\"git\"")
- return
- }
- apiToken = creds.Tokens[0]
-
- // Access to paths "/foo/bar.git/*" and "/foo/bar/.git/*" are
- // protected by the permissions on the repository named
- // "foo/bar".
- pathParts := strings.SplitN(r.URL.Path[1:], ".git/", 2)
- if len(pathParts) != 2 {
- statusCode, statusText = http.StatusNotFound, "not found"
- return
- }
- repoName := pathParts[0]
- repoName = strings.TrimRight(repoName, "/")
- httpserver.SetResponseLogFields(r.Context(), logrus.Fields{
- "repoName": repoName,
- })
-
- arv := h.clientPool.Get()
- if arv == nil {
- statusCode, statusText = http.StatusInternalServerError, "connection pool failed: "+h.clientPool.Err().Error()
- return
- }
- defer h.clientPool.Put(arv)
-
- // Log the UUID if the supplied token is a v2 token, otherwise
- // just the last five characters.
- httpserver.SetResponseLogFields(r.Context(), logrus.Fields{
- "tokenUUID": func() string {
- if strings.HasPrefix(apiToken, "v2/") && strings.IndexRune(apiToken[3:], '/') == 27 {
- // UUID part of v2 token
- return apiToken[3:30]
- } else if len(apiToken) > 5 {
- return "[...]" + apiToken[len(apiToken)-5:]
- } else {
- return apiToken
- }
- }(),
- })
-
- // Ask API server whether the repository is readable using
- // this token (by trying to read it!)
- arv.ApiToken = apiToken
- repoUUID, err := h.lookupRepo(arv, repoName)
- if err != nil {
- statusCode, statusText = http.StatusInternalServerError, err.Error()
- return
- }
- if repoUUID == "" {
- statusCode, statusText = http.StatusNotFound, "not found"
- return
- }
-
- isWrite := strings.HasSuffix(r.URL.Path, "/git-receive-pack")
- if !isWrite {
- statusText = "read"
- } else {
- err := arv.Update("repositories", repoUUID, arvadosclient.Dict{
- "repository": arvadosclient.Dict{
- "modified_at": time.Now().String(),
- },
- }, &arvadosclient.Dict{})
- if err != nil {
- statusCode, statusText = http.StatusForbidden, err.Error()
- return
- }
- statusText = "write"
- }
-
- // Regardless of whether the client asked for "/foo.git" or
- // "/foo/.git", we choose whichever variant exists in our repo
- // root, and we try {uuid}.git and {uuid}/.git first. If none
- // of these exist, we 404 even though the API told us the repo
- // _should_ exist (presumably this means the repo was just
- // created, and gitolite sync hasn't run yet).
- rewrittenPath := ""
- tryDirs := []string{
- "/" + repoUUID + ".git",
- "/" + repoUUID + "/.git",
- "/" + repoName + ".git",
- "/" + repoName + "/.git",
- }
- for _, dir := range tryDirs {
- if fileInfo, err := os.Stat(h.cluster.Git.Repositories + dir); err != nil {
- if !os.IsNotExist(err) {
- statusCode, statusText = http.StatusInternalServerError, err.Error()
- return
- }
- } else if fileInfo.IsDir() {
- rewrittenPath = dir + "/" + pathParts[1]
- break
- }
- }
- if rewrittenPath == "" {
- log.Println("WARNING:", repoUUID,
- "git directory not found in", h.cluster.Git.Repositories, tryDirs)
- // We say "content not found" to disambiguate from the
- // earlier "API says that repo does not exist" error.
- statusCode, statusText = http.StatusNotFound, "content not found"
- return
- }
- r.URL.Path = rewrittenPath
-
- h.handler.ServeHTTP(w, r)
-}
-
-var uuidRegexp = regexp.MustCompile(`^[0-9a-z]{5}-s0uqq-[0-9a-z]{15}$`)
-
-func (h *authHandler) lookupRepo(arv *arvadosclient.ArvadosClient, repoName string) (string, error) {
- reposFound := arvadosclient.Dict{}
- var column string
- if uuidRegexp.MatchString(repoName) {
- column = "uuid"
- } else {
- column = "name"
- }
- err := arv.List("repositories", arvadosclient.Dict{
- "filters": [][]string{{column, "=", repoName}},
- }, &reposFound)
- if err != nil {
- return "", err
- } else if avail, ok := reposFound["items_available"].(float64); !ok {
- return "", errors.New("bad list response from API")
- } else if avail < 1 {
- return "", nil
- } else if avail > 1 {
- return "", errors.New("name collision")
- }
- return reposFound["items"].([]interface{})[0].(map[string]interface{})["uuid"].(string), nil
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "io"
- "log"
- "net/http"
- "net/http/httptest"
- "net/url"
- "path/filepath"
- "strings"
-
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/arvadosclient"
- "git.arvados.org/arvados.git/sdk/go/arvadostest"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
- check "gopkg.in/check.v1"
-)
-
-var _ = check.Suite(&AuthHandlerSuite{})
-
-type AuthHandlerSuite struct {
- cluster *arvados.Cluster
-}
-
-func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
- arvadostest.ResetEnv()
- repoRoot, err := filepath.Abs("../api/tmp/git/test")
- c.Assert(err, check.IsNil)
-
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.Equals, nil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.Equals, nil)
-
- s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:0"}: {}}
- s.cluster.TLS.Insecure = true
- s.cluster.Git.GitCommand = "/usr/bin/git"
- s.cluster.Git.Repositories = repoRoot
-}
-
-func (s *AuthHandlerSuite) TestPermission(c *check.C) {
- client, err := arvados.NewClientFromConfig(s.cluster)
- c.Assert(err, check.IsNil)
- ac, err := arvadosclient.New(client)
- c.Assert(err, check.IsNil)
- h := &authHandler{
- cluster: s.cluster,
- clientPool: &arvadosclient.ClientPool{Prototype: ac},
- handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- log.Printf("%v", r.URL)
- io.WriteString(w, r.URL.Path)
- }),
- }
- baseURL, err := url.Parse("http://git.example/")
- c.Assert(err, check.IsNil)
- for _, trial := range []struct {
- label string
- token string
- pathIn string
- pathOut string
- status int
- }{
- {
- label: "read repo by name",
- token: arvadostest.ActiveToken,
- pathIn: arvadostest.Repository2Name + ".git/git-upload-pack",
- pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
- },
- {
- label: "read repo by uuid",
- token: arvadostest.ActiveToken,
- pathIn: arvadostest.Repository2UUID + ".git/git-upload-pack",
- pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
- },
- {
- label: "write repo by name",
- token: arvadostest.ActiveToken,
- pathIn: arvadostest.Repository2Name + ".git/git-receive-pack",
- pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
- },
- {
- label: "write repo by uuid",
- token: arvadostest.ActiveToken,
- pathIn: arvadostest.Repository2UUID + ".git/git-receive-pack",
- pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
- },
- {
- label: "uuid not found",
- token: arvadostest.ActiveToken,
- pathIn: strings.Replace(arvadostest.Repository2UUID, "6", "z", -1) + ".git/git-upload-pack",
- status: http.StatusNotFound,
- },
- {
- label: "name not found",
- token: arvadostest.ActiveToken,
- pathIn: "nonexistent-bogus.git/git-upload-pack",
- status: http.StatusNotFound,
- },
- {
- label: "read read-only repo",
- token: arvadostest.SpectatorToken,
- pathIn: arvadostest.FooRepoName + ".git/git-upload-pack",
- pathOut: arvadostest.FooRepoUUID + "/.git/git-upload-pack",
- },
- {
- label: "write read-only repo",
- token: arvadostest.SpectatorToken,
- pathIn: arvadostest.FooRepoName + ".git/git-receive-pack",
- status: http.StatusForbidden,
- },
- } {
- c.Logf("trial label: %q", trial.label)
- u, err := baseURL.Parse(trial.pathIn)
- c.Assert(err, check.IsNil)
- resp := httptest.NewRecorder()
- req := &http.Request{
- Method: "POST",
- URL: u,
- Header: http.Header{
- "Authorization": {"Bearer " + trial.token}}}
- h.ServeHTTP(resp, req)
- if trial.status == 0 {
- trial.status = http.StatusOK
- }
- c.Check(resp.Code, check.Equals, trial.status)
- if trial.status < 400 {
- if trial.pathOut != "" && !strings.HasPrefix(trial.pathOut, "/") {
- trial.pathOut = "/" + trial.pathOut
- }
- c.Check(resp.Body.String(), check.Equals, trial.pathOut)
- }
- }
-}
-
-func (s *AuthHandlerSuite) TestCORS(c *check.C) {
- h := &authHandler{cluster: s.cluster}
-
- // CORS preflight
- resp := httptest.NewRecorder()
- req := &http.Request{
- Method: "OPTIONS",
- Header: http.Header{
- "Origin": {"*"},
- "Access-Control-Request-Method": {"GET"},
- },
- }
- h.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, http.StatusOK)
- c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST")
- c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type")
- c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
- c.Check(resp.Body.String(), check.Equals, "")
-
- // CORS actual request. Bogus token and path ensure
- // authHandler responds 4xx without calling our wrapped (nil)
- // handler.
- u, err := url.Parse("git.zzzzz.arvadosapi.com/test")
- c.Assert(err, check.Equals, nil)
- resp = httptest.NewRecorder()
- req = &http.Request{
- Method: "GET",
- URL: u,
- Header: http.Header{
- "Origin": {"*"},
- "Authorization": {"OAuth2 foobar"},
- },
- }
- h.ServeHTTP(resp, req)
- c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "context"
-
- "git.arvados.org/arvados.git/lib/service"
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/arvadosclient"
- "github.com/prometheus/client_golang/prometheus"
-)
-
-var Command = service.Command(arvados.ServiceNameGitHTTP, newHandler)
-
-func newHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
- client, err := arvados.NewClientFromConfig(cluster)
- if err != nil {
- return service.ErrorHandler(ctx, cluster, err)
- }
- ac, err := arvadosclient.New(client)
- if err != nil {
- return service.ErrorHandler(ctx, cluster, err)
- }
- return &authHandler{
- clientPool: &arvadosclient.ClientPool{Prototype: ac},
- cluster: cluster,
- handler: newGitHandler(ctx, cluster),
- }
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "context"
- "net"
- "net/http"
- "net/http/cgi"
- "os"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
-)
-
-// gitHandler is an http.Handler that invokes git-http-backend (or
-// whatever backend is configured) via CGI, with appropriate
-// environment variables in place for git-http-backend or
-// gitolite-shell.
-type gitHandler struct {
- cgi.Handler
-}
-
-func newGitHandler(ctx context.Context, cluster *arvados.Cluster) http.Handler {
- const glBypass = "GL_BYPASS_ACCESS_CHECKS"
- const glHome = "GITOLITE_HTTP_HOME"
- var env []string
- path := os.Getenv("PATH")
- if cluster.Git.GitoliteHome != "" {
- env = append(env,
- glHome+"="+cluster.Git.GitoliteHome,
- glBypass+"=1")
- path = path + ":" + cluster.Git.GitoliteHome + "/bin"
- } else if home, bypass := os.Getenv(glHome), os.Getenv(glBypass); home != "" || bypass != "" {
- env = append(env, glHome+"="+home, glBypass+"="+bypass)
- ctxlog.FromContext(ctx).Printf("DEPRECATED: Passing through %s and %s environment variables. Use GitoliteHome configuration instead.", glHome, glBypass)
- }
-
- var listen arvados.URL
- for listen = range cluster.Services.GitHTTP.InternalURLs {
- break
- }
- env = append(env,
- "GIT_PROJECT_ROOT="+cluster.Git.Repositories,
- "GIT_HTTP_EXPORT_ALL=",
- "SERVER_ADDR="+listen.Host,
- "PATH="+path)
- return &gitHandler{
- Handler: cgi.Handler{
- Path: cluster.Git.GitCommand,
- Dir: cluster.Git.Repositories,
- Env: env,
- Args: []string{"http-backend"},
- },
- }
-}
-
-func (h *gitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- remoteHost, remotePort, err := net.SplitHostPort(r.RemoteAddr)
- if err != nil {
- ctxlog.FromContext(r.Context()).Errorf("Internal error: SplitHostPort(r.RemoteAddr==%q): %s", r.RemoteAddr, err)
- w.WriteHeader(http.StatusInternalServerError)
- return
- }
-
- // Copy the wrapped cgi.Handler, so these request-specific
- // variables don't leak into the next request.
- handlerCopy := h.Handler
- handlerCopy.Env = append(handlerCopy.Env,
- // In Go1.5 we can skip this, net/http/cgi will do it for us:
- "REMOTE_HOST="+remoteHost,
- "REMOTE_ADDR="+remoteHost,
- "REMOTE_PORT="+remotePort,
- // Ideally this would be a real username:
- "REMOTE_USER="+r.RemoteAddr,
- )
- handlerCopy.ServeHTTP(w, r)
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "context"
- "net/http"
- "net/http/httptest"
- "net/url"
- "regexp"
-
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
- check "gopkg.in/check.v1"
-)
-
-var _ = check.Suite(&GitHandlerSuite{})
-
-type GitHandlerSuite struct {
- cluster *arvados.Cluster
-}
-
-func (s *GitHandlerSuite) SetUpTest(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.Equals, nil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.Equals, nil)
-
- s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:80"}: {}}
- s.cluster.Git.GitoliteHome = "/test/ghh"
- s.cluster.Git.Repositories = "/"
-}
-
-func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
- u, err := url.Parse("git.zzzzz.arvadosapi.com/test")
- c.Check(err, check.Equals, nil)
- resp := httptest.NewRecorder()
- req := &http.Request{
- Method: "GET",
- URL: u,
- RemoteAddr: "[::1]:12345",
- }
- h := newGitHandler(context.Background(), s.cluster)
- h.(*gitHandler).Path = "/bin/sh"
- h.(*gitHandler).Args = []string{"-c", "printf 'Content-Type: text/plain\r\n\r\n'; env"}
-
- h.ServeHTTP(resp, req)
-
- c.Check(resp.Code, check.Equals, http.StatusOK)
- body := resp.Body.String()
- c.Check(body, check.Matches, `(?ms).*^PATH=.*:/test/ghh/bin$.*`)
- c.Check(body, check.Matches, `(?ms).*^GITOLITE_HTTP_HOME=/test/ghh$.*`)
- c.Check(body, check.Matches, `(?ms).*^GL_BYPASS_ACCESS_CHECKS=1$.*`)
- c.Check(body, check.Matches, `(?ms).*^REMOTE_HOST=::1$.*`)
- c.Check(body, check.Matches, `(?ms).*^REMOTE_PORT=12345$.*`)
- c.Check(body, check.Matches, `(?ms).*^SERVER_ADDR=`+regexp.QuoteMeta("localhost:80")+`$.*`)
-}
-
-func (s *GitHandlerSuite) TestCGIErrorOnSplitHostPortError(c *check.C) {
- u, err := url.Parse("git.zzzzz.arvadosapi.com/test")
- c.Check(err, check.Equals, nil)
- resp := httptest.NewRecorder()
- req := &http.Request{
- Method: "GET",
- URL: u,
- RemoteAddr: "test.bad.address.missing.port",
- }
- h := newGitHandler(context.Background(), s.cluster)
- h.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, http.StatusInternalServerError)
- c.Check(resp.Body.String(), check.Equals, "")
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "io/ioutil"
- "os"
- "os/exec"
- "strings"
-
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
- check "gopkg.in/check.v1"
-)
-
-var _ = check.Suite(&GitoliteSuite{})
-
-// GitoliteSuite tests need an API server, an arvados-git-httpd
-// server, and a repository hosted by gitolite.
-type GitoliteSuite struct {
- IntegrationSuite
- gitoliteHome string
-}
-
-func (s *GitoliteSuite) SetUpTest(c *check.C) {
- var err error
- s.gitoliteHome, err = ioutil.TempDir("", "githttp")
- c.Assert(err, check.Equals, nil)
-
- runGitolite := func(prog string, args ...string) {
- c.Log(prog, " ", args)
- cmd := exec.Command(prog, args...)
- cmd.Dir = s.gitoliteHome
- cmd.Env = []string{"HOME=" + s.gitoliteHome}
- for _, e := range os.Environ() {
- if !strings.HasPrefix(e, "HOME=") {
- cmd.Env = append(cmd.Env, e)
- }
- }
- diags, err := cmd.CombinedOutput()
- c.Log(string(diags))
- c.Assert(err, check.Equals, nil)
- }
-
- runGitolite("gitolite", "setup", "--admin", "root")
-
- s.tmpRepoRoot = s.gitoliteHome + "/repositories"
-
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.Equals, nil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.Equals, nil)
-
- s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:0"}: {}}
- s.cluster.TLS.Insecure = true
- s.cluster.Git.GitCommand = "/usr/share/gitolite3/gitolite-shell"
- s.cluster.Git.GitoliteHome = s.gitoliteHome
- s.cluster.Git.Repositories = s.tmpRepoRoot
-
- s.IntegrationSuite.SetUpTest(c)
-
- // Install the gitolite hooks in the bare repo we made in
- // (*IntegrationTest)SetUpTest() -- see 2.2.4 at
- // http://gitolite.com/gitolite/gitolite.html
- runGitolite("gitolite", "setup")
-}
-
-func (s *GitoliteSuite) TearDownTest(c *check.C) {
- // We really want Unsetenv here, but it's not worth forcing an
- // upgrade to Go 1.4.
- os.Setenv("GITOLITE_HTTP_HOME", "")
- os.Setenv("GL_BYPASS_ACCESS_CHECKS", "")
- if s.gitoliteHome != "" {
- err := os.RemoveAll(s.gitoliteHome)
- c.Check(err, check.Equals, nil)
- }
- s.IntegrationSuite.TearDownTest(c)
-}
-
-func (s *GitoliteSuite) TestFetch(c *check.C) {
- err := s.RunGit(c, activeToken, "fetch", "active/foo.git", "refs/heads/main")
- c.Check(err, check.Equals, nil)
-}
-
-func (s *GitoliteSuite) TestFetchUnreadable(c *check.C) {
- err := s.RunGit(c, anonymousToken, "fetch", "active/foo.git")
- c.Check(err, check.ErrorMatches, `.* not found.*`)
-}
-
-func (s *GitoliteSuite) TestPush(c *check.C) {
- err := s.RunGit(c, activeToken, "push", "active/foo.git", "main:gitolite-push")
- c.Check(err, check.Equals, nil)
-
- // Check that the commit hash appears in the gitolite log, as
- // assurance that the gitolite hooks really did run.
-
- sha1, err := exec.Command("git", "--git-dir", s.tmpWorkdir+"/.git",
- "log", "-n1", "--format=%H").CombinedOutput()
- c.Logf("git-log in workdir: %q", string(sha1))
- c.Assert(err, check.Equals, nil)
- c.Assert(len(sha1), check.Equals, 41)
-
- gitoliteLog, err := exec.Command("grep", "-r", string(sha1[:40]), s.gitoliteHome+"/.gitolite/logs").CombinedOutput()
- c.Check(err, check.Equals, nil)
- c.Logf("gitolite log message: %q", string(gitoliteLog))
-}
-
-func (s *GitoliteSuite) TestPushUnwritable(c *check.C) {
- err := s.RunGit(c, spectatorToken, "push", "active/foo.git", "main:gitolite-push-fail")
- c.Check(err, check.ErrorMatches, `.*HTTP (code = )?403.*`)
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "context"
- "errors"
- "io/ioutil"
- "os"
- "os/exec"
- "strings"
- "testing"
-
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/arvadostest"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
- check "gopkg.in/check.v1"
-)
-
-// Gocheck boilerplate
-func Test(t *testing.T) {
- check.TestingT(t)
-}
-
-// IntegrationSuite tests need an API server and an arvados-git-httpd
-// server. See GitSuite and GitoliteSuite.
-type IntegrationSuite struct {
- tmpRepoRoot string
- tmpWorkdir string
- testServer *httpserver.Server
- cluster *arvados.Cluster
-}
-
-func (s *IntegrationSuite) SetUpTest(c *check.C) {
- arvadostest.ResetEnv()
-
- var err error
- if s.tmpRepoRoot == "" {
- s.tmpRepoRoot, err = ioutil.TempDir("", "githttp")
- c.Assert(err, check.Equals, nil)
- }
- s.tmpWorkdir, err = ioutil.TempDir("", "githttp")
- c.Assert(err, check.Equals, nil)
- _, err = exec.Command("git", "init", "--bare", s.tmpRepoRoot+"/zzzzz-s0uqq-382brsig8rp3666.git").Output()
- c.Assert(err, check.Equals, nil)
- // we need git 2.28 to specify the initial branch with -b; Buster only has 2.20; so we do it in 2 steps
- _, err = exec.Command("git", "init", s.tmpWorkdir).Output()
- c.Assert(err, check.Equals, nil)
- _, err = exec.Command("sh", "-c", "cd "+s.tmpWorkdir+" && git checkout -b main").Output()
- c.Assert(err, check.Equals, nil)
- _, err = exec.Command("sh", "-c", "cd "+s.tmpWorkdir+" && echo initial >initial && git add initial && git -c user.name=Initial -c user.email=Initial commit -am 'foo: initial commit'").CombinedOutput()
- c.Assert(err, check.Equals, nil)
- _, err = exec.Command("sh", "-c", "cd "+s.tmpWorkdir+" && git push "+s.tmpRepoRoot+"/zzzzz-s0uqq-382brsig8rp3666.git main:main").CombinedOutput()
- c.Assert(err, check.Equals, nil)
- _, err = exec.Command("sh", "-c", "cd "+s.tmpWorkdir+" && echo work >work && git add work && git -c user.name=Foo -c user.email=Foo commit -am 'workdir: test'").CombinedOutput()
- c.Assert(err, check.Equals, nil)
-
- if s.cluster == nil {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.Equals, nil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.Equals, nil)
-
- s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:0"}: {}}
- s.cluster.TLS.Insecure = true
- s.cluster.Git.GitCommand = "/usr/bin/git"
- s.cluster.Git.Repositories = s.tmpRepoRoot
- s.cluster.ManagementToken = arvadostest.ManagementToken
- }
-
- s.testServer = &httpserver.Server{}
- s.testServer.Handler = httpserver.LogRequests(newHandler(context.Background(), s.cluster, "", nil))
- err = s.testServer.Start()
- c.Assert(err, check.Equals, nil)
-
- _, err = exec.Command("git", "config",
- "--file", s.tmpWorkdir+"/.git/config",
- "credential.http://"+s.testServer.Addr+"/.helper",
- "!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred").Output()
- c.Assert(err, check.Equals, nil)
- _, err = exec.Command("git", "config",
- "--file", s.tmpWorkdir+"/.git/config",
- "credential.http://"+s.testServer.Addr+"/.username",
- "none").Output()
- c.Assert(err, check.Equals, nil)
-
- // Clear ARVADOS_API_* env vars before starting up the server,
- // to make sure arvados-git-httpd doesn't use them or complain
- // about them being missing.
- os.Unsetenv("ARVADOS_API_HOST")
- os.Unsetenv("ARVADOS_API_HOST_INSECURE")
- os.Unsetenv("ARVADOS_API_TOKEN")
-}
-
-func (s *IntegrationSuite) TearDownTest(c *check.C) {
- var err error
- if s.testServer != nil {
- err = s.testServer.Close()
- }
- c.Check(err, check.Equals, nil)
- s.testServer = nil
-
- if s.tmpRepoRoot != "" {
- err = os.RemoveAll(s.tmpRepoRoot)
- c.Check(err, check.Equals, nil)
- }
- s.tmpRepoRoot = ""
-
- if s.tmpWorkdir != "" {
- err = os.RemoveAll(s.tmpWorkdir)
- c.Check(err, check.Equals, nil)
- }
- s.tmpWorkdir = ""
-
- s.cluster = nil
-}
-
-func (s *IntegrationSuite) RunGit(c *check.C, token, gitCmd, repo string, args ...string) error {
- cwd, err := os.Getwd()
- c.Assert(err, check.Equals, nil)
- defer os.Chdir(cwd)
- os.Chdir(s.tmpWorkdir)
-
- gitargs := append([]string{
- gitCmd, "http://" + s.testServer.Addr + "/" + repo,
- }, args...)
- cmd := exec.Command("git", gitargs...)
- cmd.Env = append(os.Environ(), "ARVADOS_API_TOKEN="+token)
- w, err := cmd.StdinPipe()
- c.Assert(err, check.Equals, nil)
- w.Close()
- output, err := cmd.CombinedOutput()
- c.Log("git ", gitargs, " => ", err)
- c.Log(string(output))
- if err != nil && len(output) > 0 {
- // If messages appeared on stderr, they are more
- // helpful than the err returned by CombinedOutput().
- //
- // Easier to match error strings without newlines:
- err = errors.New(strings.Replace(string(output), "\n", " // ", -1))
- }
- return err
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package githttpd
-
-import (
- "os"
- "os/exec"
-
- check "gopkg.in/check.v1"
-)
-
-var _ = check.Suite(&GitSuite{})
-
-const (
- spectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
- activeToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
- anonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
- expiredToken = "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx"
-)
-
-type GitSuite struct {
- IntegrationSuite
-}
-
-func (s *GitSuite) TestPathVariants(c *check.C) {
- s.makeArvadosRepo(c)
- for _, repo := range []string{"active/foo.git", "active/foo/.git", "arvados.git", "arvados/.git"} {
- err := s.RunGit(c, spectatorToken, "fetch", repo, "refs/heads/main")
- c.Assert(err, check.Equals, nil)
- }
-}
-
-func (s *GitSuite) TestReadonly(c *check.C) {
- err := s.RunGit(c, spectatorToken, "fetch", "active/foo.git", "refs/heads/main")
- c.Assert(err, check.Equals, nil)
- err = s.RunGit(c, spectatorToken, "push", "active/foo.git", "main:newbranchfail")
- c.Assert(err, check.ErrorMatches, `.*HTTP (code = )?403.*`)
- _, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666.git/refs/heads/newbranchfail")
- c.Assert(err, check.FitsTypeOf, &os.PathError{})
-}
-
-func (s *GitSuite) TestReadwrite(c *check.C) {
- err := s.RunGit(c, activeToken, "fetch", "active/foo.git", "refs/heads/main")
- c.Assert(err, check.Equals, nil)
- err = s.RunGit(c, activeToken, "push", "active/foo.git", "main:newbranch")
- c.Assert(err, check.Equals, nil)
- _, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666.git/refs/heads/newbranch")
- c.Assert(err, check.Equals, nil)
-}
-
-func (s *GitSuite) TestNonexistent(c *check.C) {
- err := s.RunGit(c, spectatorToken, "fetch", "thisrepodoesnotexist.git", "refs/heads/main")
- c.Assert(err, check.ErrorMatches, `.* not found.*`)
-}
-
-func (s *GitSuite) TestMissingGitdirReadableRepository(c *check.C) {
- err := s.RunGit(c, activeToken, "fetch", "active/foo2.git", "refs/heads/main")
- c.Assert(err, check.ErrorMatches, `.* not found.*`)
-}
-
-func (s *GitSuite) TestNoPermission(c *check.C) {
- for _, repo := range []string{"active/foo.git", "active/foo/.git"} {
- err := s.RunGit(c, anonymousToken, "fetch", repo, "refs/heads/main")
- c.Assert(err, check.ErrorMatches, `.* not found.*`)
- }
-}
-
-func (s *GitSuite) TestExpiredToken(c *check.C) {
- for _, repo := range []string{"active/foo.git", "active/foo/.git"} {
- err := s.RunGit(c, expiredToken, "fetch", repo, "refs/heads/main")
- c.Assert(err, check.ErrorMatches, `.* (500 while accessing|requested URL returned error: 500).*`)
- }
-}
-
-func (s *GitSuite) TestInvalidToken(c *check.C) {
- for _, repo := range []string{"active/foo.git", "active/foo/.git"} {
- err := s.RunGit(c, "s3cr3tp@ssw0rd", "fetch", repo, "refs/heads/main")
- c.Assert(err, check.ErrorMatches, `.* requested URL returned error.*`)
- }
-}
-
-func (s *GitSuite) TestShortToken(c *check.C) {
- for _, repo := range []string{"active/foo.git", "active/foo/.git"} {
- err := s.RunGit(c, "s3cr3t", "fetch", repo, "refs/heads/main")
- c.Assert(err, check.ErrorMatches, `.* (500 while accessing|requested URL returned error: 500).*`)
- }
-}
-
-func (s *GitSuite) TestShortTokenBadReq(c *check.C) {
- for _, repo := range []string{"bogus"} {
- err := s.RunGit(c, "s3cr3t", "fetch", repo, "refs/heads/main")
- c.Assert(err, check.ErrorMatches, `.*not found.*`)
- }
-}
-
-// Make a bare arvados repo at {tmpRepoRoot}/arvados.git
-func (s *GitSuite) makeArvadosRepo(c *check.C) {
- msg, err := exec.Command("git", "init", "--bare", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git").CombinedOutput()
- c.Log(string(msg))
- c.Assert(err, check.Equals, nil)
- msg, err = exec.Command("git", "--git-dir", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git", "fetch", "../../.git", "HEAD:main").CombinedOutput()
- c.Log(string(msg))
- c.Assert(err, check.Equals, nil)
-}
DispatchCloud: { ExternalURL: "" },
DispatchLSF: { ExternalURL: "" },
DispatchSLURM: { ExternalURL: "" },
- GitHTTP: { ExternalURL: "https://xxxxx.fakecluster.tld:39105/" },
- GitSSH: { ExternalURL: "" },
Health: { ExternalURL: "https://xxxxx.fakecluster.tld:42915/" },
Keepbalance: { ExternalURL: "" },
Keepproxy: { ExternalURL: "https://xxxxx.fakecluster.tld:46773/" },
COPY $workdir/fuse.conf /etc/
-COPY $workdir/gitolite.rc \
- $workdir/keep-setup.sh $workdir/common.sh $workdir/createusers.sh \
+COPY $workdir/keep-setup.sh $workdir/common.sh $workdir/createusers.sh \
$workdir/logger $workdir/runsu.sh $workdir/waitforpostgres.sh \
$workdir/yml_override.py $workdir/api-setup.sh \
$workdir/go-setup.sh $workdir/devenv.sh $workdir/cluster-config.sh $workdir/edit_users.py \
RUN sudo -u arvbox /var/lib/arvbox/service/doc/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
-RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run --only-deps
RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
blob_signing_key: $blob_signing_key
workbench_address: "https://$localip/"
websocket_address: "wss://$localip:${services[websockets-ssl]}/websocket"
- git_repo_ssh_base: "git@$localip:"
- git_repo_https_base: "http://$localip:${services[arv-git-httpd]}/"
new_users_are_active: true
auto_admin_first_user: true
auto_setup_new_users: true
ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
InternalURLs:
"http://localhost:${services[websockets]}": {}
- GitSSH:
- ExternalURL: "ssh://git@$localip:"
- GitHTTP:
- InternalURLs:
- "http://localhost:${services[arv-git-httpd]}/": {}
- ExternalURL: "https://$localip:${services[arv-git-httpd-ssl]}/"
WebDAV:
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
AutoAdminFirstUser: true
AutoSetupNewUsers: true
AutoSetupNewUsersWithVmUUID: $vm_uuid
- AutoSetupNewUsersWithRepository: true
- Git:
- GitCommand: /usr/share/gitolite3/gitolite-shell
- GitoliteHome: $ARVADOS_CONTAINER_PATH/git
- Repositories: $ARVADOS_CONTAINER_PATH/git/repositories
Volumes:
${uuid_prefix}-nyw5e-000000000000000:
Driver: Directory
[api]=8004
[controller]=8003
[controller-ssl]=8000
- [arv-git-httpd-ssl]=9000
- [arv-git-httpd]=9001
[keep-web]=9003
[keep-web-ssl]=9002
[keep-web-dl-ssl]=9004
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This is based on the default Gitolite configuration file with the following
-# changes applied as described here:
-# http://doc.arvados.org/install/install-arv-git-httpd.html
-
-# configuration variables for gitolite
-
-# This file is in perl syntax. But you do NOT need to know perl to edit it --
-# just mind the commas, use single quotes unless you know what you're doing,
-# and make sure the brackets and braces stay matched up!
-
-# (Tip: perl allows a comma after the last item in a list also!)
-
-# HELP for commands can be had by running the command with "-h".
-
-# HELP for all the other FEATURES can be found in the documentation (look for
-# "list of non-core programs shipped with gitolite" in the master index) or
-# directly in the corresponding source file.
-
-my $repo_aliases;
-my $aliases_src = "$ENV{HOME}/.gitolite/arvadosaliases.pl";
-if ($ENV{HOME} && (-e $aliases_src)) {
- $repo_aliases = do $aliases_src;
-}
-$repo_aliases ||= {};
-
-%RC = (
-
- REPO_ALIASES => $repo_aliases,
-
- # ------------------------------------------------------------------
-
- # default umask gives you perms of '0700'; see the rc file docs for
- # how/why you might change this
- UMASK => 0022,
-
- # look for "git-config" in the documentation
- GIT_CONFIG_KEYS => '',
-
- # comment out if you don't need all the extra detail in the logfile
- LOG_EXTRA => 1,
- # logging options
- # 1. leave this section as is for 'normal' gitolite logging (default)
- # 2. uncomment this line to log ONLY to syslog:
- # LOG_DEST => 'syslog',
- # 3. uncomment this line to log to syslog and the normal gitolite log:
- # LOG_DEST => 'syslog,normal',
- # 4. prefixing "repo-log," to any of the above will **also** log just the
- # update records to "gl-log" in the bare repo directory:
- # LOG_DEST => 'repo-log,normal',
- # LOG_DEST => 'repo-log,syslog',
- # LOG_DEST => 'repo-log,syslog,normal',
-
- # roles. add more roles (like MANAGER, TESTER, ...) here.
- # WARNING: if you make changes to this hash, you MUST run 'gitolite
- # compile' afterward, and possibly also 'gitolite trigger POST_COMPILE'
- ROLES => {
- READERS => 1,
- WRITERS => 1,
- },
-
- # enable caching (currently only Redis). PLEASE RTFM BEFORE USING!!!
- # CACHE => 'Redis',
-
- # ------------------------------------------------------------------
-
- # rc variables used by various features
-
- # the 'info' command prints this as additional info, if it is set
- # SITE_INFO => 'Please see http://blahblah/gitolite for more help',
-
- # the CpuTime feature uses these
- # display user, system, and elapsed times to user after each git operation
- # DISPLAY_CPU_TIME => 1,
- # display a warning if total CPU times (u, s, cu, cs) crosses this limit
- # CPU_TIME_WARN_LIMIT => 0.1,
-
- # the Mirroring feature needs this
- # HOSTNAME => "foo",
-
- # TTL for redis cache; PLEASE SEE DOCUMENTATION BEFORE UNCOMMENTING!
- # CACHE_TTL => 600,
-
- # ------------------------------------------------------------------
-
- # suggested locations for site-local gitolite code (see cust.html)
-
- # this one is managed directly on the server
- # LOCAL_CODE => "$ENV{HOME}/local",
-
- # or you can use this, which lets you put everything in a subdirectory
- # called "local" in your gitolite-admin repo. For a SECURITY WARNING
- # on this, see http://gitolite.com/gitolite/non-core.html#pushcode
- # LOCAL_CODE => "$rc{GL_ADMIN_BASE}/local",
-
- # ------------------------------------------------------------------
-
- # List of commands and features to enable
-
- ENABLE => [
-
- # COMMANDS
-
- # These are the commands enabled by default
- 'help',
- 'desc',
- 'info',
- 'perms',
- 'writable',
-
- # Uncomment or add new commands here.
- # 'create',
- # 'fork',
- # 'mirror',
- # 'readme',
- # 'sskm',
- # 'D',
-
- # These FEATURES are enabled by default.
-
- # essential (unless you're using smart-http mode)
- 'ssh-authkeys',
-
- # creates git-config enties from gitolite.conf file entries like 'config foo.bar = baz'
- 'git-config',
-
- # creates git-daemon-export-ok files; if you don't use git-daemon, comment this out
- 'daemon',
-
- # creates projects.list file; if you don't use gitweb, comment this out
- 'gitweb',
-
- # These FEATURES are disabled by default; uncomment to enable. If you
- # need to add new ones, ask on the mailing list :-)
-
- # user-visible behaviour
-
- # prevent wild repos auto-create on fetch/clone
- # 'no-create-on-read',
- # no auto-create at all (don't forget to enable the 'create' command!)
- # 'no-auto-create',
-
- # access a repo by another (possibly legacy) name
- 'Alias',
-
- # give some users direct shell access. See documentation in
- # sts.html for details on the following two choices.
- # "Shell $ENV{HOME}/.gitolite.shell-users",
- # 'Shell alice bob',
-
- # set default roles from lines like 'option default.roles-1 = ...', etc.
- # 'set-default-roles',
-
- # show more detailed messages on deny
- # 'expand-deny-messages',
-
- # show a message of the day
- # 'Motd',
-
- # system admin stuff
-
- # enable mirroring (don't forget to set the HOSTNAME too!)
- # 'Mirroring',
-
- # allow people to submit pub files with more than one key in them
- # 'ssh-authkeys-split',
-
- # selective read control hack
- # 'partial-copy',
-
- # manage local, gitolite-controlled, copies of read-only upstream repos
- # 'upstream',
-
- # updates 'description' file instead of 'gitweb.description' config item
- # 'cgit',
-
- # allow repo-specific hooks to be added
- # 'repo-specific-hooks',
-
- # performance, logging, monitoring...
-
- # be nice
- # 'renice 10',
-
- # log CPU times (user, system, cumulative user, cumulative system)
- # 'CpuTime',
-
- # syntactic_sugar for gitolite.conf and included files
-
- # allow backslash-escaped continuation lines in gitolite.conf
- # 'continuation-lines',
-
- # create implicit user groups from directory names in keydir/
- # 'keysubdirs-as-groups',
-
- # allow simple line-oriented macros
- # 'macros',
-
- # Kindergarten mode
-
- # disallow various things that sensible people shouldn't be doing anyway
- # 'Kindergarten',
- ],
-
-);
-
-# ------------------------------------------------------------------------------
-# per perl rules, this should be the last line in such a file:
-1;
-
-# Local variables:
-# mode: perl
-# End:
-# vim: set syn=perl:
+++ /dev/null
-/usr/local/lib/arvbox/logger
\ No newline at end of file
+++ /dev/null
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-. /usr/local/lib/arvbox/go-setup.sh
-
-(cd /usr/local/bin && ln -sf arvados-server arvados-git-httpd)
-
-if test "$1" = "--only-deps" ; then
- exit
-fi
-
-flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
-
-export PATH="$PATH:$ARVADOS_CONTAINER_PATH/git/bin"
-cd ~git
-exec /usr/local/bin/arvados-git-httpd
+++ /dev/null
-/usr/local/lib/arvbox/logger
\ No newline at end of file
+++ /dev/null
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -eux -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-
-if test "$1" != "--only-deps" ; then
- while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do
- sleep 1
- done
-fi
-
-mkdir -p $ARVADOS_CONTAINER_PATH/git
-
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
-
-export USER=git
-export USERNAME=git
-export LOGNAME=git
-export HOME=$ARVADOS_CONTAINER_PATH/git
-
-cd ~arvbox
-
-mkdir -p ~arvbox/.ssh ~git/.ssh
-chmod 0700 ~arvbox/.ssh ~git/.ssh
-
-if ! test -s ~arvbox/.ssh/id_rsa ; then
- ssh-keygen -t rsa -P '' -f .ssh/id_rsa
- cp ~arvbox/.ssh/id_rsa ~arvbox/.ssh/id_rsa.pub ~git/.ssh
-fi
-
-if test -s ~arvbox/.ssh/known_hosts ; then
- ssh-keygen -f ".ssh/known_hosts" -R localhost
-fi
-
-if ! test -f $ARVADOS_CONTAINER_PATH/gitolite-setup ; then
- cd ~git
-
- # Do a no-op login to populate known_hosts
- # with the hostkey, so it won't try to ask
- # about it later.
- cp .ssh/id_rsa.pub .ssh/authorized_keys
- ssh -o stricthostkeychecking=no git@localhost true
- rm .ssh/authorized_keys
-
- cp /usr/local/lib/arvbox/gitolite.rc .gitolite.rc
-
- gitolite setup -pk .ssh/id_rsa.pub
-
- if ! test -d gitolite-admin ; then
- git clone git@localhost:gitolite-admin
- fi
-
- cd gitolite-admin
- git config user.email arvados
- git config user.name arvados
- git config push.default simple
- git push
-
- touch $ARVADOS_CONTAINER_PATH/gitolite-setup
-else
- # Do a no-op login to populate known_hosts
- # with the hostkey, so it won't try to ask
- # about it later. Don't run anything,
- # get the default gitolite behavior.
- ssh -o stricthostkeychecking=no git@localhost
-fi
-
-prefix=$(arv --format=uuid user current | cut -d- -f1)
-
-if ! test -s $ARVADOS_CONTAINER_PATH/arvados-git-uuid ; then
- repo_uuid=$(arv --format=uuid repository create --repository "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"arvados\"}")
- echo $repo_uuid > $ARVADOS_CONTAINER_PATH/arvados-git-uuid
-fi
-
-repo_uuid=$(cat $ARVADOS_CONTAINER_PATH/arvados-git-uuid)
-
-if ! test -s $ARVADOS_CONTAINER_PATH/arvados-git-link-uuid ; then
- all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
-
- set +e
- read -rd $'\000' newlink <<EOF
-{
- "tail_uuid":"$all_users_group_uuid",
- "head_uuid":"$repo_uuid",
- "link_class":"permission",
- "name":"can_read"
-}
-EOF
- set -e
- link_uuid=$(arv --format=uuid link create --link "$newlink")
- echo $link_uuid > $ARVADOS_CONTAINER_PATH/arvados-git-link-uuid
-fi
-
-if ! test -d $ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git ; then
- git clone --bare /usr/src/arvados $ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git
-else
- git --git-dir=$ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git fetch -f /usr/src/arvados main:main
-fi
-
-cd /usr/src/arvados/services/api
-
-if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then
- RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env)
-else
- RAILS_ENV=development
-fi
-
-git_user_key=$(cat ~git/.ssh/id_rsa.pub)
-
-cat > config/arvados-clients.yml <<EOF
-$RAILS_ENV:
- gitolite_url: $ARVADOS_CONTAINER_PATH/git/repositories/gitolite-admin.git
- gitolite_tmp: $ARVADOS_CONTAINER_PATH/git
- arvados_api_host: $localip:${services[controller-ssl]}
- arvados_api_token: "$ARVADOS_API_TOKEN"
- arvados_api_host_insecure: false
- gitolite_arvados_git_user_key: "$git_user_key"
-EOF
-
-while true ; do
- flock $GEMLOCK bundle exec script/arvados-git-sync.rb $RAILS_ENV
- sleep 120
-done
}
}
- upstream arvados-git-httpd {
- server localhost:${services[arv-git-httpd]};
+ upstream arvados-webshell {
+ server localhost:${services[webshell]};
}
server {
- listen *:${services[arv-git-httpd-ssl]} ssl default_server;
- server_name arvados-git-httpd;
+ listen ${services[webshell-ssl]} ssl;
+ server_name arvados-webshell;
+
proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
+ proxy_read_timeout 300s;
- ssl on;
+ ssl on;
ssl_certificate "${server_cert}";
ssl_certificate_key "${server_cert_key}";
- client_max_body_size 50m;
-
- location / {
- proxy_pass http://arvados-git-httpd;
- proxy_set_header Host \$http_host;
- proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto https;
- proxy_redirect off;
- }
- }
-
-upstream arvados-webshell {
- server localhost:${services[webshell]};
-}
-server {
- listen ${services[webshell-ssl]} ssl;
- server_name arvados-webshell;
-
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- ssl on;
- ssl_certificate "${server_cert}";
- ssl_certificate_key "${server_cert_key}";
-
- location / {
- if (\$request_method = 'OPTIONS') {
- add_header 'Access-Control-Allow-Origin' '*';
- add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
- add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
- add_header 'Access-Control-Max-Age' 1728000;
- add_header 'Content-Type' 'text/plain charset=UTF-8';
- add_header 'Content-Length' 0;
- return 204;
- }
- if (\$request_method = 'POST') {
- add_header 'Access-Control-Allow-Origin' '*';
- add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
- add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
- }
- if (\$request_method = 'GET') {
- add_header 'Access-Control-Allow-Origin' '*';
- add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
- add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ location / {
+ if (\$request_method = 'OPTIONS') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ add_header 'Access-Control-Max-Age' 1728000;
+ add_header 'Content-Type' 'text/plain charset=UTF-8';
+ add_header 'Content-Length' 0;
+ return 204;
+ }
+ if (\$request_method = 'POST') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ }
+ if (\$request_method = 'GET') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ }
+
+ proxy_ssl_session_reuse off;
+ proxy_read_timeout 90;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Real-IP \$remote_addr;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_pass http://arvados-webshell;
}
-
- proxy_ssl_session_reuse off;
- proxy_read_timeout 90;
- proxy_set_header X-Forwarded-Proto https;
- proxy_set_header Host \$http_host;
- proxy_set_header X-Real-IP \$remote_addr;
- proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
- proxy_pass http://arvados-webshell;
}
}
-}
EOF
. /usr/local/lib/arvbox/common.sh
-git config --system "credential.http://$localip:${services[arv-git-httpd]}/.username" none
-git config --system "credential.http://$localip:${services[arv-git-httpd]}/.helper" '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'
-
/usr/local/lib/arvbox/runsu.sh $0-service
cd /usr/src/arvados/services/login-sync
NewUsersAreActive: true
AutoAdminFirstUser: true
AutoSetupNewUsers: true
- AutoSetupNewUsersWithRepository: true
Services:
Controller:
NewUsersAreActive: true
AutoAdminFirstUser: true
AutoSetupNewUsers: true
- AutoSetupNewUsersWithRepository: true
Services:
Controller:
NewUsersAreActive: true
AutoAdminFirstUser: true
AutoSetupNewUsers: true
- AutoSetupNewUsersWithRepository: true
Services:
Controller:
projects / arvados.git / commitdiff
