Merge branch '16534-localdb-postgresql'
authorTom Clegg <tom@tomclegg.ca>
Wed, 1 Jul 2020 03:56:40 +0000 (23:56 -0400)
committerTom Clegg <tom@tomclegg.ca>
Wed, 1 Jul 2020 03:56:40 +0000 (23:56 -0400)
refs #16534

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

25 files changed:
apps/workbench/Gemfile.lock
build/package-testing/test-package-python3-arvados-python-client.sh [new file with mode: 0755]
build/package-testing/test-package-python3-python-arvados-fuse.sh [new symlink]
build/package-testing/test-package-rh-python36-python-arvados-python-client.sh [new file with mode: 0755]
build/run-build-packages.sh
build/run-library.sh
doc/_config.yml
doc/admin/link-accounts.html.textile.liquid [new file with mode: 0644]
doc/api/methods/users.html.textile.liquid
doc/install/install-shell-server.html.textile.liquid
doc/install/install-webshell.html.textile.liquid [new file with mode: 0644]
docker/jobs/Dockerfile
docker/jobs/apt.arvados.org-dev.list
docker/jobs/apt.arvados.org-stable.list
docker/jobs/apt.arvados.org-testing.list
sdk/cwl/arvados_cwl/runner.py
sdk/cwl/tests/arvados-tests.yml
sdk/cwl/tests/wf-defaults/default-dir4.cwl
sdk/cwl/tests/wf-defaults/default-dir8.cwl [new file with mode: 0644]
sdk/cwl/tests/wf-defaults/wf4.cwl
sdk/cwl/tests/wf-defaults/wf8.cwl [new file with mode: 0644]
services/api/Gemfile.lock
services/api/lib/fix_roles_projects.rb
services/api/test/unit/group_test.rb
services/fuse/fpm-info.sh

index 2420fee24d07e056d3ee4b7047f43f87dd1b5d6d..cb4e7ab9e334cb8fdb0ae72c20ee841f4fed02b2 100644 (file)
@@ -214,7 +214,7 @@ GEM
       multi_json (~> 1.0)
       websocket-driver (>= 0.2.0)
     public_suffix (4.0.3)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-mini-profiler (1.0.2)
       rack (>= 1.2.0)
     rack-test (0.6.3)
diff --git a/build/package-testing/test-package-python3-arvados-python-client.sh b/build/package-testing/test-package-python3-arvados-python-client.sh
new file mode 100755 (executable)
index 0000000..d4e66a2
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arv-put --version
+
+/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF
+import arvados
+print("Successfully imported arvados")
+EOF
diff --git a/build/package-testing/test-package-python3-python-arvados-fuse.sh b/build/package-testing/test-package-python3-python-arvados-fuse.sh
new file mode 120000 (symlink)
index 0000000..3b9232c
--- /dev/null
@@ -0,0 +1 @@
+test-package-python27-python-arvados-fuse.sh
\ No newline at end of file
diff --git a/build/package-testing/test-package-rh-python36-python-arvados-python-client.sh b/build/package-testing/test-package-rh-python36-python-arvados-python-client.sh
new file mode 100755 (executable)
index 0000000..1a69256
--- /dev/null
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arv-put --version
+
+/usr/share/python3/dist/rh-python36-python-arvados-python-client/bin/python3 << EOF
+import arvados
+print("Successfully imported arvados")
+EOF
index 3ba1dcc05e8776fc57a205e2deb79a0224a8e370..862b93e6e4667ad561c478e13b0f6f53ee2d012f 100755 (executable)
@@ -334,6 +334,9 @@ fpm_build_virtualenv "libpam-arvados" "sdk/pam"
 # The FUSE driver
 fpm_build_virtualenv "arvados-fuse" "services/fuse"
 
+# The FUSE driver - Python3 package
+fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
+
 # The node manager
 fpm_build_virtualenv "arvados-node-manager" "services/nodemanager"
 
index b75b6cca78bae2f6e11fa309c08b921f874f2ae5..f8e5129daeb0ce63aba2230c04214ca252bba476 100755 (executable)
@@ -574,7 +574,7 @@ fpm_build_virtualenv () {
   cd build/usr/share/$python/dist/$PYTHON_PKG/
 
   # Replace the shebang lines in all python scripts, and handle the activate
-  # scripts too This is a functional replacement of the 237 line
+  # scripts too. This is a functional replacement of the 237 line
   # virtualenv_tools.py script that doesn't work in python3 without serious
   # patching, minus the parts we don't need (modifying pyc files, etc).
   for binfile in `ls bin/`; do
@@ -632,7 +632,7 @@ fpm_build_virtualenv () {
     COMMAND_ARR+=('--rpm-auto-add-directories')
   fi
 
-  if [[ "$PKG" == "arvados-python-client" ]]; then
+  if [[ "$PKG" == "arvados-python-client" ]] || [[ "$PKG" == "arvados-fuse" ]]; then
     if [[ "$python" == "python2.7" ]]; then
       COMMAND_ARR+=('--conflicts' "$PYTHON3_PKG_PREFIX-$PKG")
     else
index 3b59cbca45205983ba4b83429f06b914946a53dd..be52a204c02d4e9548eeaa1139ff8126cff4f400 100644 (file)
@@ -153,8 +153,9 @@ navbar:
       - admin/index.html.textile.liquid
     - Users and Groups:
       - admin/user-management.html.textile.liquid
-      - admin/reassign-ownership.html.textile.liquid
       - admin/user-management-cli.html.textile.liquid
+      - admin/reassign-ownership.html.textile.liquid
+      - admin/link-accounts.html.textile.liquid
       - admin/group-management.html.textile.liquid
       - admin/federation.html.textile.liquid
       - admin/merge-remote-account.html.textile.liquid
@@ -217,6 +218,7 @@ navbar:
       - install/install-ws.html.textile.liquid
       - install/install-arv-git-httpd.html.textile.liquid
       - install/install-shell-server.html.textile.liquid
+      - install/install-webshell.html.textile.liquid
     - Containers API:
       - install/crunch2-slurm/install-compute-node.html.textile.liquid
       - install/install-jobs-image.html.textile.liquid
diff --git a/doc/admin/link-accounts.html.textile.liquid b/doc/admin/link-accounts.html.textile.liquid
new file mode 100644 (file)
index 0000000..d0ac6a0
--- /dev/null
@@ -0,0 +1,48 @@
+---
+layout: default
+navsection: admin
+title: "Link user accounts"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+If a user needs to log in to Arvados with a upstream account or provider, they may end up with two Arvados user accounts.  If the user still has the ability to log in with the old account, they can use the "self-serve account linking":{{site.baseurl}}/user/topics/link-accounts.html feature of workbench.  However, if the user does not have the ability to log in with both upstream accounts, the admin can also link the accounts using the command line.
+
+h3. Step 1: Determine user uuids
+
+User uuids can be determined by browsing workbench or using @arv user list@ at the command line.
+
+Account linking works by recording in the database that a log in to the "old" account should redirected and treated as a login to the "new" account.
+
+The "old" account is the Arvados account that will be redirected.
+
+The "new" account is the user that the "old" account is redirected to.  As part of account linking any Arvados records owned by the "old" account is also transferred to the "new" account.
+
+Counter-intuitively, if you do not want the account uuid of the user to change, the "new" account should be the pre-existing account, and the "old" account should be the redundant second account that was more recently created.  This means "old" and "new" are opposite from their expected chronological meaning.  In this case, the use of "old" and "new" reflect the direction of transfer of ownership -- the login was associated with the "old" user account, but will be associated with the "new" user account.
+
+In the example below, @zzzzz-tpzed-3kz0nwtjehhl0u4@ is the "old" account (the pre-existing account we want to keep) and @zzzzz-tpzed-fr97h9t4m5jffxs@ is the "new" account (the redundant account we want to merge into the existing account).
+
+h3. Step 2: Create a project
+
+Create a project owned by the "new" account that will hold any data owned by the "old" account.
+
+<pre>
+$ arv --format=uuid group create --group '{"group_class": "project", "name": "Data from old user", "owner_uuid": "zzzzz-tpzed-fr97h9t4m5jffxs"}'
+zzzzz-j7d0g-mczqiguhil13083
+</pre>
+
+h3. Step 3: Merge "old" user to "new" user
+
+The @user merge@ method redirects login and reassigns data from the "old" account to the "new" account.
+
+<pre>
+$ arv user merge  --redirect-to-new-user \
+  --old-user-uuid=zzzzz-tpzed-3kz0nwtjehhl0u4 \
+  --new-user-uuid=zzzzz-tpzed-fr97h9t4m5jffxs \
+  --new-owner-uuid=zzzzz-j7d0g-mczqiguhil13083 \
+</pre>
+
+Note that authorization credentials (API tokens, ssh keys) are also transferred to the "new" account, so credentials used to access the "old" account work with the "new" account.
index 4c33f2afe820df5e662622b5880a9fd75f3561a6..cde189d6ffa341833cadd7cd08be32fd79146a7c 100644 (file)
@@ -154,3 +154,21 @@ Arguments:
 table(table table-bordered table-condensed).
 |_. Argument |_. Type |_. Description |_. Location |_. Example |
 {background:#ccffcc}.|uuid|string|The UUID of the User in question.|path||
+
+h3. merge
+
+Transfer ownership of data from the "old" user account to the "new" user account.  When @redirect_to_new_user@ is @true@ this also causes logins to the "old" account to be redirected to the "new" account.  The "old" user account that was redirected becomes invisible in user listings.
+
+See "Merge user accounts":{{site.baseurl}}/admin/link-accounts.html , "Reassign user data ownership":{{site.baseurl}}/admin/reassign-ownership.html and "Linking alternate login accounts":{{site.baseurl}}/user/topics/link-accounts.html for examples of how this method is used.
+
+Must supply either @new_user_token@ (the currently authorized user will be the "old" user), or both @new_user_uuid@ and @old_user_uuid@ (the currently authorized user must be an admin).
+
+Arguments:
+
+table(table table-bordered table-condensed).
+|_. Argument |_. Type |_. Description |_. Location |_. Example |
+|new_user_token|string|A valid token for the "new" user|query||
+|new_user_uuid|uuid|The uuid of the "new" account|query||
+|old_user_uuid|uuid|The uuid of the "old" account|query||
+|new_owner_uuid|uuid|The uuid of a project to which objects owned by the "old" user will be reassigned.|query||
+|redirect_to_new_user|boolean|If true, also redirect login and reassign authorization credentials from "old" user to the "new" user|query||
index 44b3834ab84ec8df76d4810c1ee76dbaf7fa0845..5ac5e9e6b870a2753287b2b8a59e50c6686d80df 100644 (file)
@@ -69,7 +69,7 @@ As an Arvados admin user (such as the system root user), create a "scoped token"
 
 <notextile>
 <pre>
-<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'
+<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'</span>
 {
  ...
  "api_token":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz",
diff --git a/doc/install/install-webshell.html.textile.liquid b/doc/install/install-webshell.html.textile.liquid
new file mode 100644 (file)
index 0000000..4040fcf
--- /dev/null
@@ -0,0 +1,184 @@
+---
+layout: default
+navsection: installguide
+title: Configure webshell
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Introduction":#introduction
+# "Prerequisites":#prerequisites
+# "Update config.yml":#configure
+# "Update nginx configuration":#update-nginx
+# "Install packages":#install-packages
+# "Configure shellinabox":#config-shellinabox
+# "Configure pam":#config-pam
+# "Confirm working installation":#confirm-working
+
+h2(#introduction). Introduction
+
+Arvados supports @webshell@, which allows ssh access to shell nodes via the browser. This functionality is integrated in @Workbench@.
+
+@Webshell@ is provided by the @shellinabox@ package which runs on each shell node for which webshell is enabled. For authentication, a supported @pam library@ that allows authentication against Arvados is also required. One Nginx (or similar web server) virtualhost is also needed to expose all the @shellinabox@ instances via https.
+
+h2(#prerequisites). Prerequisites
+
+# "Install workbench":{{site.baseurl}}/install/install-workbench-app.html
+# "Set up a shell node":{{site.baseurl}}/install/install-shell-server.html
+
+h2(#configure). Update config.yml
+
+Edit the cluster config at @config.yml@ and set @Services.WebShell.ExternalURL@.  Replace @zzzzz@ with your cluster id. Workbench will use this information to activate its support for webshell.
+
+<notextile>
+<pre><code>    Services:
+      WebShell:
+        InternalURLs: {}
+        ExternalURL: <span class="userinput">https://webshell.ClusterID.example.com/</span>
+</span></code></pre>
+</notextile>
+
+h2(#update-nginx). Update Nginx configuration
+
+The arvados-webshell service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption. This Nginx virtualhost could live on your Workbench server, or any other server that is reachable by your Workbench users and can access the @shell-in-a-box@ service on the shell node(s) on port 4200.
+
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-webshell.conf@ with the following configuration.  Options that need attention are marked in <span class="userinput">red</span>.
+
+<notextile><pre>
+upstream arvados-webshell {
+  server                <span class="userinput">shell.ClusterID.example.com</span>:<span class="userinput">4200</span>;
+}
+
+server {
+  listen                443 ssl;
+  server_name           webshell.<span class="userinput">ClusterID.example.com</span>;
+
+  proxy_connect_timeout 90s;
+  proxy_read_timeout    300s;
+
+  ssl                   on;
+  ssl_certificate       <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+  ssl_certificate_key   <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
+
+  location /<span class="userinput">shell.ClusterID</span> {
+    if ($request_method = 'OPTIONS') {
+       add_header 'Access-Control-Allow-Origin' '*'; 
+       add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+       add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+       add_header 'Access-Control-Max-Age' 1728000;
+       add_header 'Content-Type' 'text/plain charset=UTF-8';
+       add_header 'Content-Length' 0;
+       return 204;
+    }
+    if ($request_method = 'POST') {
+       add_header 'Access-Control-Allow-Origin' '*';
+       add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+       add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+    }
+    if ($request_method = 'GET') {
+       add_header 'Access-Control-Allow-Origin' '*';
+       add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+       add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+    }
+
+    proxy_ssl_session_reuse off;
+    proxy_read_timeout  90;
+    proxy_set_header    X-Forwarded-Proto https;
+    proxy_set_header    Host $http_host;
+    proxy_set_header    X-Real-IP $remote_addr;
+    proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_pass          http://arvados-webshell;
+  }
+}
+</pre></notextile>
+
+Note that the location line in the nginx config matches your shell node hostname *without domain*, because that is how the shell node was defined in the "Set up a shell node":{{site.baseurl}}/install/install-shell-server.html#vm-record instructions. It makes for a more user friendly experience in Workbench.
+
+For additional shell nodes with @shell-in-a-box@, add @location@ and @upstream@ sections as needed.
+
+{% assign arvados_component = 'shellinabox libpam-arvados' %}
+
+{% include 'install_packages' %}
+
+h2(#config-shellinabox). Configure shellinabox
+
+h3. Red Hat and Centos
+
+Edit @/etc/sysconfig/shellinaboxd@:
+
+<notextile><pre>
+# TCP port that shellinboxd's webserver listens on
+PORT=4200
+
+# SSL is disabled because it is terminated in Nginx. Adjust as needed.
+OPTS="--disable-ssl --no-beep --service=/<span class="userinput">shell.ClusterID.example.com</span>:AUTH:HOME:SHELL"
+</pre></notextile>
+
+<notextile>
+<pre>
+<code># <span class="userinput">systemctl enable shellinabox</span></code>
+<code># <span class="userinput">systemctl start shellinabox</span></code>
+</pre>
+</notextile>
+
+h3. Debian and Ubuntu
+
+Edit @/etc/default/shellinabox@:
+
+<notextile><pre>
+# TCP port that shellinboxd's webserver listens on
+SHELLINABOX_PORT=4200
+
+# SSL is disabled because it is terminated in Nginx. Adjust as needed.
+SHELLINABOX_ARGS="--disable-ssl --no-beep --service=/<span class="userinput">shell.ClusterID.example.com</span>:AUTH:HOME:SHELL"
+</pre></notextile>
+
+<notextile>
+<pre>
+<code># <span class="userinput">systemctl enable shellinabox</span></code>
+<code># <span class="userinput">systemctl start shellinabox</span></code>
+</pre>
+</notextile>
+
+
+h2(#config-pam). Configure pam
+
+Use a text editor to create a new file @/etc/pam.d/shellinabox@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
+
+<notextile><pre>
+# This example is a stock debian "login" file with libpam_arvados
+# replacing pam_unix, and the "noprompt" option in use. It can be
+# installed as /etc/pam.d/shellinabox .
+
+auth       optional   pam_faildelay.so  delay=3000000
+auth [success=ok new_authtok_reqd=ok ignore=ignore user_unknown=bad default=die] pam_securetty.so
+auth       requisite  pam_nologin.so
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so close
+session       required   pam_env.so readenv=1
+session       required   pam_env.so readenv=1 envfile=/etc/default/locale
+
+auth [success=1 default=ignore] pam_python.so /usr/lib/security/libpam_arvados.py <span class="userinput">ClusterID.example.com</span> <span class="userinput">shell.ClusterID.example.com</span> noprompt
+auth    requisite            pam_deny.so
+auth    required            pam_permit.so
+
+auth       optional   pam_group.so
+session    required   pam_limits.so
+session    optional   pam_lastlog.so
+session    optional   pam_motd.so  motd=/run/motd.dynamic
+session    optional   pam_motd.so
+session    optional   pam_mail.so standard
+
+@include common-account
+@include common-session
+@include common-password
+
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so open
+</pre></notextile>
+
+h2(#confirm-working). Confirm working installation
+
+A user should be able to log in to the shell server, using webshell via workbench. Please refer to "Accessing an Arvados VM with Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html
+
index 876ac4f9f49cea14c42f54f1ebe37423b4251cd2..15993c4bc322619e125ddb5411a79a2d0f4348f0 100644 (file)
@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Based on Debian Stretch
-FROM debian:stretch-slim
-MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
+FROM debian:buster-slim
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
 
 ENV DEBIAN_FRONTEND noninteractive
 
index 468000ed29b9244460e28f0b5abe5f5efd13f133..4de87397bca754a57e384c3155d88b82a30983fc 100644 (file)
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch-dev main
+deb http://apt.arvados.org/ buster-dev main
index afbc51effe84979f49f5d1c9584bf951c2408922..7882afd01c96235b1fde32767d56a68aeada8d03 100644 (file)
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch main
+deb http://apt.arvados.org/ buster main
index c8ea91d070a572365006e849015d48006d060a22..3bb599087eaf513bb5c3f6dc2e32d54108d3db53 100644 (file)
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch-testing main
+deb http://apt.arvados.org/ buster-testing main
index 7bb66a158e50646f1fc984df7ea30ccd31528bf3..71e499ebcab0cca29ccbee7a350cfbbb5aaa6e19 100644 (file)
@@ -169,21 +169,47 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
         #
         # Found a file, check for secondaryFiles
         #
-        primary["secondaryFiles"] = []
+        specs = []
+        primary["secondaryFiles"] = secondaryspec
         for i, sf in enumerate(aslist(secondaryspec)):
             pattern = builder.do_eval(sf["pattern"], context=primary)
             if pattern is None:
                 continue
+            if isinstance(pattern, list):
+                specs.extend(pattern)
+            elif isinstance(pattern, dict):
+                specs.append(pattern)
+            elif isinstance(pattern, str):
+                specs.append({"pattern": pattern})
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
+        found = []
+        for i, sf in enumerate(specs):
+            if isinstance(sf, dict):
+                if sf.get("class") == "File":
+                    pattern = sf["basename"]
+                else:
+                    pattern = sf["pattern"]
+                    required = sf.get("required")
+            elif isinstance(sf, str):
+                pattern = sf
+                required = True
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
             sfpath = substitute(primary["location"], pattern)
-            required = builder.do_eval(sf.get("required"), context=primary)
+            required = builder.do_eval(required, context=primary)
 
             if fsaccess.exists(sfpath):
-                primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+                found.append({"location": sfpath, "class": "File"})
             elif required:
                 raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
                     "Required secondary file '%s' does not exist" % sfpath)
 
-        primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+        primary["secondaryFiles"] = cmap(found)
         if discovered is not None:
             discovered[primary["location"]] = primary["secondaryFiles"]
     elif inputschema["type"] not in primitive_types_set:
@@ -434,9 +460,13 @@ def packed_workflow(arvrunner, tool, merged_map):
     def visit(v, cur_id):
         if isinstance(v, dict):
             if v.get("class") in ("CommandLineTool", "Workflow"):
-                if "id" not in v:
-                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
-                cur_id = rewrite_to_orig.get(v["id"], v["id"])
+                if tool.metadata["cwlVersion"] == "v1.0" and "id" not in v:
+                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field, add an 'id' or use to cwlVersion: v1.1")
+                if "id" in v:
+                    cur_id = rewrite_to_orig.get(v["id"], v["id"])
+            if "path" in v and "location" not in v:
+                v["location"] = v["path"]
+                del v["path"]
             if "location" in v and not v["location"].startswith("keep:"):
                 v["location"] = merged_map[cur_id].resolved[v["location"]]
             if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
index df9fac8426cc450f0dc7014c72d0e799863657cb..c4c0968756a46b04ad8b201cbc66241fb4d6826d 100644 (file)
   output:
     out: null
   tool: wf-defaults/wf4.cwl
-  doc: default in embedded subworkflow missing 'id' field
+  doc: default in embedded subworkflow missing 'id' field, v1.0
   should_fail: true
 
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf8.cwl
+  doc: default in embedded subworkflow missing 'id' field, v1.1
+  should_fail: false
+
 - job: null
   output:
     out: null
index 8bfc5d63f744a784e13d78ec48049211ae629c48..bd927824886d1805cf8daf260e4911e6b4fe2d85 100644 (file)
@@ -21,4 +21,4 @@ steps:
             class: Directory
             location: inp1
       outputs: []
-      arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
+      arguments: [echo, $(inputs.inp2)]
diff --git a/sdk/cwl/tests/wf-defaults/default-dir8.cwl b/sdk/cwl/tests/wf-defaults/default-dir8.cwl
new file mode 100644 (file)
index 0000000..a5b9c2f
--- /dev/null
@@ -0,0 +1,24 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.1
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+steps:
+  step1:
+    in: []
+    out: []
+    run:
+      class: CommandLineTool
+      inputs:
+        inp2:
+          type: Directory
+          default:
+            class: Directory
+            location: inp1
+      outputs: []
+      arguments: [echo, $(inputs.inp2)]
index 6e562e43dbd791f390dd25f6803e4a23c49ce967..3f498fdffbfa56100c721f6efb78efcb40267f74 100644 (file)
@@ -14,4 +14,4 @@ steps:
   step1:
     in: []
     out: []
-    run: default-dir4.cwl
\ No newline at end of file
+    run: default-dir4.cwl
diff --git a/sdk/cwl/tests/wf-defaults/wf8.cwl b/sdk/cwl/tests/wf-defaults/wf8.cwl
new file mode 100644 (file)
index 0000000..2548fae
--- /dev/null
@@ -0,0 +1,17 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.1
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+requirements:
+  SubworkflowFeatureRequirement: {}
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir8.cwl
index c8a1a27b79c1e939438dcb8ed603c206a299c409..127a09ee2db71a00bc7c05ee5e2e651ea379a33d 100644 (file)
@@ -180,7 +180,7 @@ GEM
     pg (1.1.4)
     power_assert (1.1.4)
     public_suffix (4.0.3)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-test (0.6.3)
       rack (>= 1.0)
     rails (5.0.7.2)
index 5dd127b3e230246962b179f2431d9ad49a95483a..5bb013c9add7a1f241d4779768cef462ac9956b2 100644 (file)
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+require 'update_permissions'
+
 include CurrentApiClient
 
 def fix_roles_projects
@@ -20,10 +22,11 @@ def fix_roles_projects
 
           if old_owner != system_user_uuid
             # 2) Ownership of a role becomes a can_manage link
-            Link.create!(link_class: 'permission',
+            Link.new(link_class: 'permission',
                          name: 'can_manage',
                          tail_uuid: old_owner,
-                         head_uuid: g.uuid)
+                         head_uuid: g.uuid).
+              save!(validate: false)
           end
         end
 
@@ -37,20 +40,22 @@ def fix_roles_projects
           # 3) If a role owns anything, give it to system user and it
           # becomes a can_manage link
           klass.joins("join groups on groups.uuid=#{klass.table_name}.owner_uuid and groups.group_class='role'").each do |owned|
-            Link.create!(link_class: 'permission',
-                         name: 'can_manage',
-                         tail_uuid: owned.owner_uuid,
-                         head_uuid: owned.uuid)
+            Link.new(link_class: 'permission',
+                     name: 'can_manage',
+                     tail_uuid: owned.owner_uuid,
+                     head_uuid: owned.uuid).
+              save!(validate: false)
             owned.owner_uuid = system_user_uuid
             owned.save_with_unique_name!
           end
         end
 
         Group.joins("join groups as g2 on g2.uuid=groups.owner_uuid and g2.group_class='role'").each do |owned|
-          Link.create!(link_class: 'permission',
+          Link.new(link_class: 'permission',
                        name: 'can_manage',
                        tail_uuid: owned.owner_uuid,
-                       head_uuid: owned.uuid)
+                       head_uuid: owned.uuid).
+            save!(validate: false)
           owned.owner_uuid = system_user_uuid
           owned.save_with_unique_name!
         end
index 3d1fda927f0554ce7955a4e73d7e0e8921f7d8a5..30fddfa5b8be8f89c2f43651c1c316a6e59253fe 100644 (file)
@@ -6,6 +6,7 @@ require 'test_helper'
 require 'fix_roles_projects'
 
 class GroupTest < ActiveSupport::TestCase
+  include DbCurrentTime
 
   test "cannot set owner_uuid to object with existing ownership cycle" do
     set_user_from_auth :active_trustedclient
@@ -317,6 +318,14 @@ insert into groups (uuid, owner_uuid, name, group_class, created_at, updated_at)
     g6 = insert_group Group.generate_uuid, system_user_uuid, 'name collision', 'role'
     g7 = insert_group Group.generate_uuid, users(:active).uuid, 'name collision', 'role'
 
+    g8 = insert_group Group.generate_uuid, users(:active).uuid, 'trashed with no class', nil
+    g8obj = Group.find_by_uuid(g8)
+    g8obj.trash_at = db_current_time
+    g8obj.delete_at = db_current_time
+    act_as_system_user do
+      g8obj.save!(validate: false)
+    end
+
     refresh_permissions
 
     act_as_system_user do
@@ -328,6 +337,7 @@ update links set tail_uuid='#{g5}' where uuid='#{l1.uuid}'
     end
 
     assert_equal nil, Group.find_by_uuid(g1).group_class
+    assert_equal nil, Group.find_by_uuid(g8).group_class
     assert_equal users(:active).uuid, Group.find_by_uuid(g2).owner_uuid
     assert_equal g3, Group.find_by_uuid(g4).owner_uuid
     assert !Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
@@ -337,6 +347,7 @@ update links set tail_uuid='#{g5}' where uuid='#{l1.uuid}'
     fix_roles_projects
 
     assert_equal 'role', Group.find_by_uuid(g1).group_class
+    assert_equal 'role', Group.find_by_uuid(g8).group_class
     assert_equal system_user_uuid, Group.find_by_uuid(g2).owner_uuid
     assert_equal system_user_uuid, Group.find_by_uuid(g4).owner_uuid
     assert Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
index fd94ef7afa3340edea84a486e4abd03810fe1b8d..f789abe69270c024e73a5294666bc06169b45026 100644 (file)
@@ -9,6 +9,6 @@ case "$TARGET" in
         fpm_depends+=(fuse-libs)
         ;;
     debian* | ubuntu*)
-        fpm_depends+=(libcurl3-gnutls libpython2.7)
+        fpm_depends+=(libcurl3-gnutls)
         ;;
 esac